<s> Loads [ <NUM_LIT> ] = DAG . getExtLoad ( Ext , DL , <STR_LIT> , Chain , Ptr , LoadNode -> getPointerInfo ( ) , <STR_LIT> , LoadNode -> isVolatile ( ) , LoadNode -> isNonTemporal ( ) , LoadNode -> isInvariant ( ) , Alignment ) ; Tmp1 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Loads [ <NUM_LIT> ] , ShiftAmount ) ; SDValue Tmp4 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Tmp1 , Loads [ <NUM_LIT> ] ) ; Result = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Tmp4 , Tmp2 ) ; LoadChain = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Loads [ <NUM_LIT> ] . getValue ( <NUM_LIT> ) , Loads [ <NUM_LIT> ] . getValue ( <NUM_LIT> ) , Loads [ <NUM_LIT> ] . getValue ( <NUM_LIT> ) , Loads [ <NUM_LIT> ] . getValue ( <NUM_LIT> ) ) ; } else { Result = DAG . getLoad ( <STR_LIT> , DL , Chain , Ptr , LoadNode -> getPointerInfo ( ) , LoadNode -> isVolatile ( ) , LoadNode -> isNonTemporal ( ) , LoadNode -> isInvariant ( ) , LoadNode -> getAlignment ( ) ) ; LoadChain = Result . getValue ( <NUM_LIT> ) ; } } else llvm_unreachable ( <STR_LIT> ) ; Result = DAG . getNode ( <STR_LIT> , DL , VT , Result ) ; SDValue Ops [ ] = { Result , LoadChain } ;

return DAG . getMergeValues ( Ops , DL ) ;


<s> if ( Alignment == <NUM_LIT> ) { SDValue Loads [ <NUM_LIT> ] ; Loads [ <NUM_LIT> ] = DAG . getExtLoad ( Ext , DL , <STR_LIT> , Chain , Base , LoadNode -> getPointerInfo ( ) , <STR_LIT> , LoadNode -> isVolatile ( ) , LoadNode -> isNonTemporal ( ) , LoadNode -> isInvariant ( ) , Alignment ) ; SDValue Increment = DAG . getConstant ( <NUM_LIT> , <STR_LIT> ) ; Ptr = DAG . getNode ( <STR_LIT> , DL , Base . getValueType ( ) , Base , Increment ) ; Loads [ <NUM_LIT> ] = DAG . getExtLoad ( Ext , DL , <STR_LIT> , Chain , Ptr , LoadNode -> getPointerInfo ( ) , <STR_LIT> , LoadNode -> isVolatile ( ) , LoadNode -> isNonTemporal ( ) , LoadNode -> isInvariant ( ) , Alignment ) ; SDValue ShiftAmount = DAG . getConstant ( <NUM_LIT> , <STR_LIT> ) ; SDValue Tmp1 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Loads [ <NUM_LIT> ] , ShiftAmount ) ; SDValue Tmp2 = DAG . getNode ( <STR_LIT> , DL , <STR_LIT> , Tmp1 , Loads [ <NUM_LIT> ] ) ; Increment = DAG . getConstant ( <NUM_LIT> , <STR_LIT> ) ;


<s> case <STR_LIT> : case <STR_LIT> : return LowerVECTOR_SHIFT ( Op , DAG ) ; case <STR_LIT> : return LowerConstantPool ( Op , DAG ) ; case <STR_LIT> : return LowerEH_RETURN ( Op , DAG ) ; case <STR_LIT> : return LowerRETURNADDR ( Op , DAG ) ; case <STR_LIT> : return LowerFRAMEADDR ( Op , DAG ) ; case <STR_LIT> : llvm_unreachable ( <STR_LIT> ) ; case <STR_LIT> : return LowerATOMIC_FENCE ( Op , DAG ) ; case <STR_LIT> : return LowerGLOBALADDRESS ( Op , DAG ) ; case <STR_LIT> : return LowerBlockAddress ( Op , DAG ) ; case <STR_LIT> : return LowerVASTART ( Op , DAG ) ;

case <STR_LIT> : return LowerBR_JT ( Op , DAG ) ;


<s> if ( Lane == - <NUM_LIT> ) Lane = <NUM_LIT> ; if ( Lane == <NUM_LIT> && V1 . getOpcode ( ) == <STR_LIT> ) return createSplat ( DAG , dl , VT , V1 . getOperand ( <NUM_LIT> ) ) ; if ( Lane == <NUM_LIT> && V1 . getOpcode ( ) == <STR_LIT> && ! isa < ConstantSDNode > ( V1 . getOperand ( <NUM_LIT> ) ) ) { bool IsScalarToVector = true ; for ( unsigned i = <NUM_LIT> , e = V1 . getNumOperands ( ) ; i != e ; ++ i ) if ( V1 . getOperand ( i ) . getOpcode ( ) != <STR_LIT> ) { IsScalarToVector = false ; break ;


<s> if ( i == First ) continue ; if ( Elem [ i ] == Elem [ First ] || Elem [ i ] . getOpcode ( ) == <STR_LIT> ) continue ; IsSplat = false ; break ; } if ( IsSplat ) return DAG . getNode ( <STR_LIT> , dl , VecTy , Elem [ First ] ) ; } if ( AllConst ) { uint64_t Val = <NUM_LIT> ; unsigned W = ElemTy . getSizeInBits ( ) ; uint64_t Mask = ( ElemTy == <STR_LIT> ) ? <NUM_LIT> : ( ElemTy == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i != Num ; ++ i ) Val = ( Val << W ) | ( Consts [ i ] -> getZExtValue ( ) & Mask ) ; SDValue V0 = DAG . getConstant ( Val , dl , <STR_LIT> ) ; return DAG . getBitcast ( VecTy , V0 ) ; } MVT HalfTy = <STR_LIT> ( ElemTy , Num / <NUM_LIT> ) ; SDValue L = ( ElemTy == <STR_LIT> ) ? Elem [ <NUM_LIT> ] : buildVector32 ( { Elem . data ( ) , Num / <NUM_LIT> } , dl , HalfTy , DAG ) ; SDValue H = ( ElemTy == <STR_LIT> ) ? Elem [ <NUM_LIT> ] : buildVector32 ( { Elem . data ( ) + Num / <NUM_LIT> , Num / <NUM_LIT> } , dl , HalfTy , DAG ) ; unsigned Id = <STR_LIT> ;

SDNode * N = DAG . getMachineNode ( TargetOpcode :: REG_SEQUENCE , dl , VecTy , { DAG . getTargetConstant ( Id , dl , <STR_LIT> ) , L , DAG . getTargetConstant ( <STR_LIT> , dl , <STR_LIT> ) , H , DAG . getTargetConstant ( <STR_LIT> , dl , <STR_LIT> ) } ) ;


<s> if ( Elem [ i ] == Elem [ First ] || Elem [ i ] . getOpcode ( ) == <STR_LIT> ) continue ; IsSplat = false ; break ; } if ( IsSplat ) return DAG . getNode ( <STR_LIT> , dl , VecTy , Elem [ First ] ) ; } if ( AllConst ) { uint64_t Val = <NUM_LIT> ; unsigned W = ElemTy . getSizeInBits ( ) ; uint64_t Mask = ( ElemTy == <STR_LIT> ) ? <NUM_LIT> : ( ElemTy == <STR_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; for ( unsigned i = <NUM_LIT> ; i != Num ; ++ i ) Val = ( Val << W ) | ( Consts [ i ] -> getZExtValue ( ) & Mask ) ; SDValue V0 = DAG . getConstant ( Val , dl , <STR_LIT> ) ; return DAG . getBitcast ( VecTy , V0 ) ; } MVT HalfTy = <STR_LIT> ( ElemTy , Num / <NUM_LIT> ) ; SDValue L = ( ElemTy == <STR_LIT> ) ? Elem [ <NUM_LIT> ] : buildVector32 ( { Elem . data ( ) , Num / <NUM_LIT> } , dl , HalfTy , DAG ) ; SDValue H = ( ElemTy == <STR_LIT> ) ? Elem [ <NUM_LIT> ] : buildVector32 ( { Elem . data ( ) + Num / <NUM_LIT> , Num / <NUM_LIT> } , dl , HalfTy , DAG ) ; unsigned Id = <STR_LIT> ; SDNode * N = DAG . getMachineNode ( TargetOpcode :: REG_SEQUENCE , dl , VecTy , { DAG . getTargetConstant ( Id , dl , <STR_LIT> ) , L , DAG . getTargetConstant ( <STR_LIT> , dl , <STR_LIT> ) , H , DAG . getTargetConstant ( <STR_LIT> , dl , <STR_LIT> ) } ) ; return SDValue ( N , <NUM_LIT> ) ;


<s> State . addLoc ( CCValAssign :: getMem ( ValNo , ValVT , Offset , LocVT , LocInfo ) ) ; return false ; } if ( HST . useHVX64BOps ( ) && ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> ) ) { if ( unsigned Reg = State . AllocateReg ( VecLstD ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT , Reg , LocVT , LocInfo ) ) ; return false ; } unsigned Offset = State . AllocateStack ( <NUM_LIT> , <NUM_LIT> ) ; State . addLoc ( CCValAssign :: getMem ( ValNo , ValVT , Offset , LocVT , LocInfo ) ) ; return false ; } if ( HST . useHVX128BOps ( ) && ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> ) ) { if ( unsigned Reg = State . AllocateReg ( VecLstD ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT , Reg , LocVT , LocInfo ) ) ; return false ; } unsigned Offset = State . AllocateStack ( <NUM_LIT> , <NUM_LIT> ) ;


<s> static bool CC_<ISA_LIT>Vector ( unsigned ValNo , MVT ValVT , MVT LocVT , CCValAssign :: LocInfo LocInfo , <STR_LIT> ArgFlags , CCState & State ) { static const MCPhysReg VecLstS [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; static const MCPhysReg VecLstD [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; auto & MF = State . getMachineFunction ( ) ; auto & HST = MF . getSubtarget < <ISA_LIT>Subtarget > ( ) ; if ( HST . useHVX64BOps ( ) && ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> ) ) { if ( unsigned Reg = State . AllocateReg ( VecLstS ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT , Reg , LocVT , LocInfo ) ) ; return false ; } unsigned Offset = State . AllocateStack ( <NUM_LIT> , <NUM_LIT> ) ; State . addLoc ( CCValAssign :: getMem ( ValNo , ValVT , Offset , LocVT , LocInfo ) ) ; return false ; } if ( HST . useHVX64BOps ( ) && ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> ) ) { if ( unsigned Reg = State . AllocateReg ( VecLstD ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT , Reg , LocVT , LocInfo ) ) ; return false ; } unsigned Offset = State . AllocateStack ( <NUM_LIT> , <NUM_LIT> ) ; State . addLoc ( CCValAssign :: getMem ( ValNo , ValVT , Offset , LocVT , LocInfo ) ) ; return false ; } if ( HST . useHVX128BOps ( ) && ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> ) ) { if ( unsigned Reg = State . AllocateReg ( VecLstD ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT , Reg , LocVT , LocInfo ) ) ; return false ; } unsigned Offset = State . AllocateStack ( <NUM_LIT> , <NUM_LIT> ) ; State . addLoc ( CCValAssign :: getMem ( ValNo , ValVT , Offset , LocVT , LocInfo ) ) ; return false ;


<s> case <STR_LIT> : case <STR_LIT> : if ( Subtarget . hasV60TOps ( ) && Subtarget . useHVXOps ( ) && Subtarget . useHVX128BOps ( ) ) RRC = & <STR_LIT> ; else RRC = & <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> case <STR_LIT> : RRC = & <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : if ( Subtarget . hasV60TOps ( ) && Subtarget . useHVXOps ( ) && Subtarget . useHVX128BOps ( ) ) RRC = & <STR_LIT> ; else RRC = & <STR_LIT> ; break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> :


<s> auto & HST = static_cast < const <ISA_LIT>Subtarget & > ( DAG . getSubtarget ( ) ) ; bool ValidHVX128BType = HST . useHVX128BOps ( ) && ( VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> ) ; bool ValidHVXType = HST . useHVX64BOps ( ) && ( VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> ) ; if ( ValidHVX128BType || ValidHVXType || VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> || VT == <STR_LIT> ) { IsInc = ( Ptr -> getOpcode ( ) == <STR_LIT> ) ; Base = Ptr -> getOperand ( <NUM_LIT> ) ; Offset = Ptr -> getOperand ( <NUM_LIT> ) ; return isa < ConstantSDNode > ( Offset ) ; } return false ;


<s> for ( unsigned i = <NUM_LIT> , e = Op . getNumOperands ( ) ; i != e ; ++ i ) Ops . push_back ( Op . getOperand ( i ) ) ; if ( BW == <NUM_LIT> ) return buildVector32 ( Ops , SDLoc ( Op ) , VT , DAG ) ;

return buildVector64 ( Ops , SDLoc ( Op ) , VT , DAG ) ;


<s> if ( BW == <NUM_LIT> || BW == <NUM_LIT> ) { SmallVector < SDValue , <NUM_LIT> > Ops ; for ( unsigned i = <NUM_LIT> , e = Op . getNumOperands ( ) ; i != e ; ++ i ) Ops . push_back ( Op . getOperand ( i ) ) ; if ( BW == <NUM_LIT> ) return buildVector32 ( Ops , SDLoc ( Op ) , VT , DAG ) ; return buildVector64 ( Ops , SDLoc ( Op ) , VT , DAG ) ;


<s> assert ( ( Width == <NUM_LIT> * <NUM_LIT> && Subtarget . useHVX64BOps ( ) ) || ( Width == <NUM_LIT> * <NUM_LIT> && Subtarget . useHVX128BOps ( ) ) ) ; SDValue Vec1 = Op . getOperand ( <NUM_LIT> ) ; MVT OpTy = Subtarget . useHVX64BOps ( ) ? <STR_LIT> : <STR_LIT> ; MVT ReTy = Subtarget . useHVX64BOps ( ) ? <STR_LIT> : <STR_LIT> ; SDValue B0 = DAG . getNode ( <STR_LIT> , dl , OpTy , Vec0 ) ; SDValue B1 = DAG . getNode ( <STR_LIT> , dl , OpTy , Vec1 ) ; SDValue VC = DAG . getNode ( <STR_LIT> , dl , ReTy , B1 , B0 ) ; return DAG . getNode ( <STR_LIT> , dl , VT , VC ) ; } } if ( VT . getSizeInBits ( ) != <NUM_LIT> && VT . getSizeInBits ( ) != <NUM_LIT> ) return SDValue ( ) ; SDValue C0 = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; SDValue C32 = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; SDValue W = DAG . getConstant ( Width , dl , <STR_LIT> ) ; SDValue S = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , W , C32 ) ; SDValue V = C0 ; for ( unsigned i = <NUM_LIT> , e = NElts ; i != e ; ++ i ) { unsigned N = NElts - i - <NUM_LIT> ; SDValue OpN = Op . getOperand ( N ) ; if ( VT . getSizeInBits ( ) == <NUM_LIT> && OpN . getValueSizeInBits ( ) == <NUM_LIT> ) { SDValue C = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; OpN = DAG . getNode ( <STR_LIT> , dl , VT , C , OpN ) ; } SDValue Idx = DAG . getConstant ( N , dl , <STR_LIT> ) ; SDValue Offset = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Idx , W ) ; SDValue Or = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , S , Offset ) ; if ( VT . getSizeInBits ( ) == <NUM_LIT> ) V = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , { V , OpN , Or } ) ; else if ( VT . getSizeInBits ( ) == <NUM_LIT> ) V = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , { V , OpN , Or } ) ; else return SDValue ( ) ; } return DAG . getNode ( <STR_LIT> , dl , VT , V ) ;


<s> MVT ST = VecVT . getSimpleVT ( ) ; if ( ST == <STR_LIT> || ST == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , dl , VT , Op . getOperand ( <NUM_LIT> ) , Vec0 ) ; if ( UseHVX ) { assert ( ( Width == <NUM_LIT> * <NUM_LIT> && Subtarget . useHVX64BOps ( ) ) || ( Width == <NUM_LIT> * <NUM_LIT> && Subtarget . useHVX128BOps ( ) ) ) ; SDValue Vec1 = Op . getOperand ( <NUM_LIT> ) ; MVT OpTy = Subtarget . useHVX64BOps ( ) ? <STR_LIT> : <STR_LIT> ; MVT ReTy = Subtarget . useHVX64BOps ( ) ? <STR_LIT> : <STR_LIT> ; SDValue B0 = DAG . getNode ( <STR_LIT> , dl , OpTy , Vec0 ) ; SDValue B1 = DAG . getNode ( <STR_LIT> , dl , OpTy , Vec1 ) ; SDValue VC = DAG . getNode ( <STR_LIT> , dl , ReTy , B1 , B0 ) ; return DAG . getNode ( <STR_LIT> , dl , VT , VC ) ; } } if ( VT . getSizeInBits ( ) != <NUM_LIT> && VT . getSizeInBits ( ) != <NUM_LIT> ) return SDValue ( ) ; SDValue C0 = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; SDValue C32 = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; SDValue W = DAG . getConstant ( Width , dl , <STR_LIT> ) ; SDValue S = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , W , C32 ) ;


<s> EVT VT = Op . getOperand ( <NUM_LIT> ) . getValueType ( ) ; SDLoc dl ( Op ) ; bool UseHVX = Subtarget . useHVXOps ( ) ; bool UseHVX64B = Subtarget . useHVX64BOps ( ) ; if ( ! VT . isVector ( ) || ! UseHVX ) return SDValue ( ) ; EVT ResVT = Op . getValueType ( ) ; unsigned ResSize = ResVT . getSizeInBits ( ) ; unsigned VectorSizeInBits = UseHVX64B ? ( <NUM_LIT> * <NUM_LIT> ) : ( <NUM_LIT> * <NUM_LIT> ) ; unsigned OpSize = VT . getSizeInBits ( ) ; if ( ! ( ResVT . isByteSized ( ) && ResSize == VectorSizeInBits ) || ! ( VT . isByteSized ( ) && OpSize == <NUM_LIT> * VectorSizeInBits ) ) return SDValue ( ) ; ConstantSDNode * Cst = dyn_cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ; if ( ! Cst ) return SDValue ( ) ; unsigned Val = Cst -> getZExtValue ( ) ; if ( Val == <NUM_LIT> ) { SDValue Vec = Op . getOperand ( <NUM_LIT> ) ;

return DAG . getTargetExtractSubreg ( <STR_LIT> , dl , ResVT , Vec ) ;


<s> MVT LocVT = <STR_LIT> ( SVT . getSizeInBits ( ) ) ; SDValue VecCast = DAG . getNode ( <STR_LIT> , dl , LocVT , Vec ) ; SDValue Shifted = DAG . getNode ( <STR_LIT> , dl , LocVT , VecCast , Offset ) ; return DAG . getNode ( <STR_LIT> , dl , LocVT , Shifted , DAG . getConstant ( <NUM_LIT> , dl , LocVT ) ) ; } else if ( W == <NUM_LIT> ) { unsigned Subreg = ( X == <NUM_LIT> ) ? <STR_LIT> : <NUM_LIT> ; if ( X == <NUM_LIT> ) Subreg = <STR_LIT> ; else if ( SVT == <STR_LIT> && X == <NUM_LIT> ) Subreg = <STR_LIT> ; else if ( SVT == <STR_LIT> && X == <NUM_LIT> ) Subreg = <STR_LIT> ; else if ( SVT == <STR_LIT> && X == <NUM_LIT> ) Subreg = <STR_LIT> ; else llvm_unreachable ( <STR_LIT> ) ; N = DAG . getTargetExtractSubreg ( Subreg , dl , <STR_LIT> , Vec ) ; } else if ( SVT . getSizeInBits ( ) == <NUM_LIT> ) { N = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Ops ) ; } else if ( SVT . getSizeInBits ( ) == <NUM_LIT> ) { N = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Ops ) ; if ( VT . getSizeInBits ( ) == <NUM_LIT> ) N = DAG . getTargetExtractSubreg ( <STR_LIT> , dl , <STR_LIT> , N ) ; } else return SDValue ( ) ; return DAG . getNode ( <STR_LIT> , dl , VT , N ) ; } SDValue Offset = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Idx , DAG . getConstant ( EltSize , dl , <STR_LIT> ) ) ; SDValue Shifted = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Width , DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ) ; SDValue Combined = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Shifted , Offset ) ; const SDValue Ops [ ] = { Vec , Combined } ; SDValue N ; if ( VecVT . getSizeInBits ( ) == <NUM_LIT> ) { N = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Ops ) ; } else { N = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Ops ) ; if ( VT . getSizeInBits ( ) == <NUM_LIT> ) N = DAG . getTargetExtractSubreg ( <STR_LIT> , dl , <STR_LIT> , N ) ; } return DAG . getNode ( <STR_LIT> , dl , VT , N ) ;


<s> LocVT = <STR_LIT> ; LocInfo = CCValAssign :: BCvt ; } else if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { LocVT = <STR_LIT> ; LocInfo = CCValAssign :: BCvt ; } else if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { LocVT = <STR_LIT> ; ValVT = <STR_LIT> ; LocInfo = CCValAssign :: Full ; } else if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || ( LocVT == <STR_LIT> && HST . useHVX128BOps ( ) ) ) { LocVT = <STR_LIT> ; ValVT = <STR_LIT> ; LocInfo = CCValAssign :: Full ; } else if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { LocVT = <STR_LIT> ; ValVT = <STR_LIT> ; LocInfo = CCValAssign :: Full ; } if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { if ( ! RetCC_<ISA_LIT>32 ( ValNo , ValVT , LocVT , LocInfo , ArgFlags , State ) ) return false ; } if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { if ( ! RetCC_<ISA_LIT>64 ( ValNo , ValVT , LocVT , LocInfo , ArgFlags , State ) ) return false ; } if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { if ( ! RetCC_<ISA_LIT>Vector ( ValNo , ValVT , LocVT , LocInfo , ArgFlags , State ) ) return false ; } return true ;


<s> LocVT = <STR_LIT> ; ValVT = <STR_LIT> ; LocInfo = CCValAssign :: Full ; } else if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || ( LocVT == <STR_LIT> && HST . useHVX128BOps ( ) ) ) { LocVT = <STR_LIT> ; ValVT = <STR_LIT> ; LocInfo = CCValAssign :: Full ; } else if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { LocVT = <STR_LIT> ; ValVT = <STR_LIT> ; LocInfo = CCValAssign :: Full ; } if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { if ( ! RetCC_<ISA_LIT>32 ( ValNo , ValVT , LocVT , LocInfo , ArgFlags , State ) ) return false ; } if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { if ( ! RetCC_<ISA_LIT>64 ( ValNo , ValVT , LocVT , LocInfo , ArgFlags , State ) ) return false ; } if ( LocVT == <STR_LIT> || LocVT == <STR_LIT> || LocVT == <STR_LIT> ) { if ( ! RetCC_<ISA_LIT>Vector ( ValNo , ValVT , LocVT , LocInfo , ArgFlags , State ) ) return false ; }


<s> static bool CC_<ISA_LIT>32 ( unsigned ValNo , MVT ValVT , MVT LocVT , CCValAssign :: LocInfo LocInfo , <STR_LIT> ArgFlags , CCState & State ) { static const unsigned RegList [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; if ( unsigned Reg = State . AllocateReg ( RegList , <NUM_LIT> ) ) { State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT , Reg , LocVT , LocInfo ) ) ;


<s> static bool CC_<ISA_LIT>32 ( unsigned ValNo , MVT ValVT , MVT LocVT , CCValAssign :: LocInfo LocInfo , <STR_LIT> ArgFlags , CCState & State ) { static const unsigned RegList [ ] = { <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> , <STR_LIT> } ; if ( unsigned Reg = State . AllocateReg ( RegList , <NUM_LIT> ) ) {

State . addLoc ( CCValAssign :: getReg ( ValNo , ValVT , Reg , LocVT , LocInfo ) ) ;


<s> MachineFunction * MF = BB -> getParent ( ) ; <ISA_LIT>MachineFunctionInfo * FuncInfo = MF -> getInfo < <ISA_LIT>MachineFunctionInfo > ( ) ; FuncInfo -> addAllocaAdjustInst ( MI ) ; return BB ; } default :


<s> case 'r' : switch ( VT . getSimpleVT ( ) . SimpleTy ) { default : assert ( <NUM_LIT> && <STR_LIT> ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return std :: make_pair ( <NUM_LIT> , <STR_LIT> ) ;


<s> std :: pair < unsigned , const TargetRegisterClass * > <ISA_LIT>TargetLowering :: getRegForInlineAsmConstraint ( const std :: string & Constraint , EVT VT ) const { if ( Constraint . size ( ) == <NUM_LIT> ) { switch ( Constraint [ <NUM_LIT> ] ) { case 'r' :

switch ( VT . getSimpleVT ( ) . SimpleTy ) {


<s> default : assert ( <NUM_LIT> && <STR_LIT> ) ; case <STR_LIT> : return LowerRETURNADDR ( Op , DAG ) ; case <STR_LIT> : return LowerFRAMEADDR ( Op , DAG ) ; case <STR_LIT> : assert ( <NUM_LIT> && <STR_LIT> ) ; case <STR_LIT> : return LowerMEMBARRIER ( Op , DAG ) ; case <STR_LIT> : return LowerATOMIC_FENCE ( Op , DAG ) ; case <STR_LIT> : return LowerGLOBALADDRESS ( Op , DAG ) ; case <STR_LIT> : return LowerVASTART ( Op , DAG ) ; case <STR_LIT> : return LowerBR_JT ( Op , DAG ) ; case <STR_LIT> : return LowerDYNAMIC_STACKALLOC ( Op , DAG ) ; case <STR_LIT> : return LowerSELECT_CC ( Op , DAG ) ; case <STR_LIT> : return LowerINTRINSIC_WO_CHAIN ( Op , DAG ) ; case <STR_LIT> : return LowerINLINEASM ( Op , DAG ) ;


<s> SDValue ArgAdjust = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Sub , DAG . getConstant ( <NUM_LIT> , <STR_LIT> ) ) ; const <ISA_LIT>RegisterInfo * QRI = static_cast < const <ISA_LIT>RegisterInfo * > ( DAG . getTarget ( ) . getRegisterInfo ( ) ) ; SDValue CopyChain = DAG . getCopyToReg ( Chain , dl , QRI -> getStackRegister ( ) , Sub ) ; SDValue Ops [ <NUM_LIT> ] = { ArgAdjust , CopyChain } ;

return DAG . getMergeValues ( Ops , dl ) ;


<s> const <ISA_LIT>RegisterInfo * QRI = static_cast < const <ISA_LIT>RegisterInfo * > ( DAG . getTarget ( ) . getRegisterInfo ( ) ) ; SDValue CopyChain = DAG . getCopyToReg ( Chain , dl , QRI -> getStackRegister ( ) , Sub ) ; SDValue Ops [ <NUM_LIT> ] = { ArgAdjust , CopyChain } ; return DAG . getMergeValues ( Ops , dl ) ;


<s> MFI -> setFrameAddressIsTaken ( true ) ; EVT VT = Op . getValueType ( ) ; SDLoc dl ( Op ) ; unsigned Depth = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; SDValue FrameAddr = DAG . getCopyFromReg ( DAG . getEntryNode ( ) , dl , TRI -> getFrameRegister ( ) , VT ) ; while ( Depth -- ) FrameAddr = DAG . getLoad ( VT , dl , DAG . getEntryNode ( ) , FrameAddr , MachinePointerInfo ( ) , false , false , false , <NUM_LIT> ) ;


<s> SDLoc dl ( Op ) ; unsigned Depth = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; SDValue FrameAddr = DAG . getCopyFromReg ( DAG . getEntryNode ( ) , dl , TRI -> getFrameRegister ( ) , VT ) ; while ( Depth -- ) FrameAddr = DAG . getLoad ( VT , dl , DAG . getEntryNode ( ) , FrameAddr , MachinePointerInfo ( ) , false , false , false , <NUM_LIT> ) ; return FrameAddr ;


<s> switch ( Node -> getOpcode ( ) ) { case <STR_LIT> : { unsigned NumOps = Node -> getNumOperands ( ) ; if ( Node -> getOperand ( NumOps - <NUM_LIT> ) . getValueType ( ) == <STR_LIT> ) -- NumOps ; for ( unsigned i = InlineAsm :: Op_FirstOperand ; i != NumOps ; ) { if ( FuncInfo -> hasClobberLR ( ) ) break ; unsigned Flags = cast < ConstantSDNode > ( Node -> getOperand ( i ) ) -> getZExtValue ( ) ; unsigned NumVals = InlineAsm :: getNumOperandRegisters ( Flags ) ; ++ i ; switch ( InlineAsm :: getKind ( Flags ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case InlineAsm :: Kind_RegDef : case InlineAsm :: Kind_RegUse : case InlineAsm :: Kind_Imm : case InlineAsm :: Kind_Clobber : case InlineAsm :: Kind_Mem : { for ( ; NumVals ; -- NumVals , ++ i ) { } break ; } case InlineAsm :: Kind_RegDefEarlyClobber : { for ( ; NumVals ; -- NumVals , ++ i ) { unsigned Reg = cast < RegisterSDNode > ( Node -> getOperand ( i ) ) -> getReg ( ) ; const <ISA_LIT>RegisterInfo * QRI = static_cast < const <ISA_LIT>RegisterInfo * > ( DAG . getTarget ( ) . getRegisterInfo ( ) ) ; if ( Reg == QRI -> getRARegister ( ) ) { FuncInfo -> setHasClobberLR ( true ) ; break ; } } break ;


<s> unsigned Flags = cast < ConstantSDNode > ( Node -> getOperand ( i ) ) -> getZExtValue ( ) ; unsigned NumVals = InlineAsm :: getNumOperandRegisters ( Flags ) ; ++ i ; switch ( InlineAsm :: getKind ( Flags ) ) { default : llvm_unreachable ( <STR_LIT> ) ; case InlineAsm :: Kind_RegDef : case InlineAsm :: Kind_RegUse : case InlineAsm :: Kind_Imm : case InlineAsm :: Kind_Clobber : case InlineAsm :: Kind_Mem : { for ( ; NumVals ; -- NumVals , ++ i ) { } break ;


<s> MachineFrameInfo * MFI = MF . getFrameInfo ( ) ; MFI -> setReturnAddressIsTaken ( true ) ; if ( verifyReturnAddressArgumentIsConstant ( Op , DAG ) ) return SDValue ( ) ; EVT VT = Op . getValueType ( ) ; SDLoc dl ( Op ) ; unsigned Depth = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; if ( Depth ) { SDValue FrameAddr = LowerFRAMEADDR ( Op , DAG ) ; SDValue Offset = DAG . getConstant ( <NUM_LIT> , <STR_LIT> ) ; return DAG . getLoad ( VT , dl , DAG . getEntryNode ( ) , DAG . getNode ( <STR_LIT> , dl , VT , FrameAddr , Offset ) , MachinePointerInfo ( ) , false , false , false , <NUM_LIT> ) ;


<s> const TargetRegisterInfo * TRI = DAG . getTarget ( ) . getRegisterInfo ( ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; MachineFrameInfo * MFI = MF . getFrameInfo ( ) ; MFI -> setReturnAddressIsTaken ( true ) ; if ( verifyReturnAddressArgumentIsConstant ( Op , DAG ) ) return SDValue ( ) ; EVT VT = Op . getValueType ( ) ; SDLoc dl ( Op ) ; unsigned Depth = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; if ( Depth ) { SDValue FrameAddr = LowerFRAMEADDR ( Op , DAG ) ; SDValue Offset = DAG . getConstant ( <NUM_LIT> , <STR_LIT> ) ; return DAG . getLoad ( VT , dl , DAG . getEntryNode ( ) , DAG . getNode ( <STR_LIT> , dl , VT , FrameAddr , Offset ) , MachinePointerInfo ( ) , false , false , false , <NUM_LIT> ) ; } unsigned Reg = MF . addLiveIn ( TRI -> getRARegister ( ) , getRegClassFor ( <STR_LIT> ) ) ; return DAG . getCopyFromReg ( DAG . getEntryNode ( ) , dl , Reg , VT ) ;


<s> int Scale = AM . Scale ; if ( Scale < <NUM_LIT> ) Scale = - Scale ; switch ( Scale ) { case <NUM_LIT> : break ;


<s> } SDValue JumpTableBase = DAG . getNode ( <STR_LIT> , dl , getPointerTy ( ) , TargetJT ) ; SDValue ShiftIndex = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Index , DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ) ; SDValue JTAddress = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , JumpTableBase , ShiftIndex ) ; SDValue LoadTarget = DAG . getLoad ( <STR_LIT> , dl , Chain , JTAddress , MachinePointerInfo ( ) , false , false , false , <NUM_LIT> ) ; return DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Chain , LoadTarget ) ;


<s> SDValue StoreAddr = DAG . getNode ( <STR_LIT> , dl , getPointerTy ( ) , DAG . getRegister ( <STR_LIT> , getPointerTy ( ) ) , DAG . getIntPtrConstant ( <NUM_LIT> , dl ) ) ; Chain = DAG . getStore ( Chain , dl , Handler , StoreAddr , MachinePointerInfo ( ) , false , false , <NUM_LIT> ) ;

Chain = DAG . getCopyToReg ( Chain , dl , OffsetReg , Offset ) ;


<s> bool <ISA_LIT>TargetLowering :: mayBeEmittedAsTailCall ( CallInst * CI ) const { if ( ! CI -> isTailCall ( ) || HTM . Options . DisableTailCalls ) return false ;


<s> if ( ! CI -> isTailCall ( ) || HTM . Options . DisableTailCalls ) return false ; return true ;


<s> SDValue <ISA_LIT>TargetLowering :: GetDynamicTLSAddr ( SelectionDAG & DAG , SDValue Chain , GlobalAddressSDNode * GA , SDValue * InFlag , EVT PtrVT , unsigned ReturnReg , unsigned char OperandFlags ) const { MachineFrameInfo & MFI = DAG . getMachineFunction ( ) . getFrameInfo ( ) ; SDVTList NodeTys = DAG . getVTList ( <STR_LIT> , <STR_LIT> ) ; SDLoc dl ( GA ) ; SDValue TGA = DAG . getTargetGlobalAddress ( GA -> getGlobal ( ) , dl , GA -> getValueType ( <NUM_LIT> ) , GA -> getOffset ( ) , OperandFlags ) ; if ( InFlag ) { SDValue Ops [ ] = { Chain , TGA , DAG . getRegister ( <STR_LIT> , PtrVT ) , * InFlag } ; Chain = DAG . getNode ( <STR_LIT> , dl , NodeTys , Ops ) ; } else { SDValue Ops [ ] = { Chain , TGA , DAG . getRegister ( <STR_LIT> , PtrVT ) } ; Chain = DAG . getNode ( <STR_LIT> , dl , NodeTys , Ops ) ; }


<s> case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : break ; } return nullptr ;


<s> if ( ! isa < GlobalAddressSDNode > ( Callee ) && ! isa < ExternalSymbolSDNode > ( Callee ) ) { return false ; } if ( ! CCMatch ) return false ; if ( isVarArg ) return false ; if ( isCalleeStructRet || isCallerStructRet ) return false ;


<s> if ( even_start == ( int ) ( mask_len * <NUM_LIT> ) - <NUM_LIT> ) return StridedLoadKind :: Even ; for ( auto idx : Mask ) { if ( ( idx - odd_start ) == <NUM_LIT> ) odd_start = idx ; else break ;


<s> if ( ST == <STR_LIT> || ST == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , dl , VT , Op . getOperand ( <NUM_LIT> ) , Vec0 ) ; if ( UseHVX ) { assert ( ( Width == <NUM_LIT> * <NUM_LIT> && Subtarget . useHVXSglOps ( ) ) || ( Width == <NUM_LIT> * <NUM_LIT> && Subtarget . useHVXDblOps ( ) ) ) ; SDValue Vec1 = Op . getOperand ( <NUM_LIT> ) ; MVT OpTy = Subtarget . useHVXSglOps ( ) ? <STR_LIT> : <STR_LIT> ; MVT ReTy = Subtarget . useHVXSglOps ( ) ? <STR_LIT> : <STR_LIT> ; SDValue B0 = DAG . getNode ( <STR_LIT> , dl , OpTy , Vec0 ) ; SDValue B1 = DAG . getNode ( <STR_LIT> , dl , OpTy , Vec1 ) ; SDValue VC = DAG . getNode ( <STR_LIT> , dl , ReTy , B1 , B0 ) ; return DAG . getNode ( <STR_LIT> , dl , VT , VC ) ; } } if ( VT . getSizeInBits ( ) != <NUM_LIT> && VT . getSizeInBits ( ) != <NUM_LIT> ) return SDValue ( ) ; SDValue C0 = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; SDValue C32 = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; SDValue W = DAG . getConstant ( Width , dl , <STR_LIT> ) ; SDValue S = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , W , C32 ) ; SDValue V = C0 ; for ( unsigned i = <NUM_LIT> , e = NElts ; i != e ; ++ i ) { unsigned N = NElts - i - <NUM_LIT> ; SDValue OpN = Op . getOperand ( N ) ; if ( VT . getSizeInBits ( ) == <NUM_LIT> && OpN . getValueType ( ) . getSizeInBits ( ) == <NUM_LIT> ) { SDValue C = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; OpN = DAG . getNode ( <STR_LIT> , dl , VT , C , OpN ) ; } SDValue Idx = DAG . getConstant ( N , dl , <STR_LIT> ) ; SDValue Offset = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Idx , W ) ; SDValue Or = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , S , Offset ) ; if ( VT . getSizeInBits ( ) == <NUM_LIT> ) V = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , { V , OpN , Or } ) ;

else if ( VT . getSizeInBits ( ) == <NUM_LIT> ) V = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , { V , OpN , Or } ) ;


<s> SDLoc dl ( Op ) ; bool UseHVX = Subtarget . useHVXOps ( ) ; bool UseHVXSgl = Subtarget . useHVXSglOps ( ) ; if ( ! VT . isVector ( ) || ! UseHVX ) return SDValue ( ) ; EVT ResVT = Op . getValueType ( ) ; unsigned ResSize = ResVT . getSizeInBits ( ) ; unsigned VectorSizeInBits = UseHVXSgl ? ( <NUM_LIT> * <NUM_LIT> ) : ( <NUM_LIT> * <NUM_LIT> ) ; unsigned OpSize = VT . getSizeInBits ( ) ; if ( ! ( ResVT . isByteSized ( ) && ResSize == VectorSizeInBits ) || ! ( VT . isByteSized ( ) && OpSize == <NUM_LIT> * VectorSizeInBits ) ) return SDValue ( ) ; ConstantSDNode * Cst = dyn_cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) ;


<s> if ( ConstantSDNode * CI = dyn_cast < ConstantSDNode > ( Idx ) ) { uint64_t X = CI -> getZExtValue ( ) ; SDValue Offset = DAG . getConstant ( X * EltSize , dl , <STR_LIT> ) ; const SDValue Ops [ ] = { Vec , Width , Offset } ; ConstantSDNode * CW = dyn_cast < ConstantSDNode > ( Width ) ; assert ( CW && <STR_LIT> ) ; SDValue N ; MVT SVT = VecVT . getSimpleVT ( ) ; uint64_t W = CW -> getZExtValue ( ) ; if ( W == <NUM_LIT> ) { unsigned Subreg = ( X == <NUM_LIT> ) ? <STR_LIT> : <NUM_LIT> ; if ( X == <NUM_LIT> ) Subreg = <STR_LIT> ; else if ( SVT == <STR_LIT> && X == <NUM_LIT> ) Subreg = <STR_LIT> ; else if ( SVT == <STR_LIT> && X == <NUM_LIT> ) Subreg = <STR_LIT> ; else if ( SVT == <STR_LIT> && X == <NUM_LIT> ) Subreg = <STR_LIT> ; else llvm_unreachable ( <STR_LIT> ) ; N = DAG . getTargetExtractSubreg ( Subreg , dl , <STR_LIT> , Vec ) ; } else if ( SVT . getSizeInBits ( ) == <NUM_LIT> ) { N = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Ops ) ; } else if ( SVT . getSizeInBits ( ) == <NUM_LIT> ) { N = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Ops ) ; if ( VT . getSizeInBits ( ) == <NUM_LIT> ) N = DAG . getTargetExtractSubreg ( <STR_LIT> , dl , <STR_LIT> , N ) ; } else return SDValue ( ) ; return DAG . getNode ( <STR_LIT> , dl , VT , N ) ; } SDValue Offset = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Idx , DAG . getConstant ( EltSize , dl , <STR_LIT> ) ) ; SDValue Shifted = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Width , DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ) ; SDValue Combined = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Shifted , Offset ) ; const SDValue Ops [ ] = { Vec , Combined } ; SDValue N ; if ( VecVT . getSizeInBits ( ) == <NUM_LIT> ) { N = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Ops ) ; } else { N = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Ops ) ; if ( VT . getSizeInBits ( ) == <NUM_LIT> ) N = DAG . getTargetExtractSubreg ( <STR_LIT> , dl , <STR_LIT> , N ) ;


<s> return DAG . getNode ( <STR_LIT> , dl , VT , N ) ; } SDValue Offset = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Idx , DAG . getConstant ( EltSize , dl , <STR_LIT> ) ) ; SDValue Shifted = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Width , DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ) ; SDValue Combined = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Shifted , Offset ) ; if ( VT . getSizeInBits ( ) == <NUM_LIT> && Val . getValueType ( ) . getSizeInBits ( ) == <NUM_LIT> ) { SDValue C = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; Val = DAG . getNode ( <STR_LIT> , dl , VT , C , Val ) ; }

const SDValue Ops [ ] = { Vec , Val , Combined } ;


<s> SDValue Offset = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Idx , DAG . getConstant ( EltSize , dl , <STR_LIT> ) ) ; SDValue Shifted = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Width , DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ) ; SDValue Combined = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Shifted , Offset ) ; if ( VT . getSizeInBits ( ) == <NUM_LIT> && Val . getValueType ( ) . getSizeInBits ( ) == <NUM_LIT> ) { SDValue C = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; Val = DAG . getNode ( <STR_LIT> , dl , VT , C , Val ) ; } const SDValue Ops [ ] = { Vec , Val , Combined } ; SDValue N ;

if ( VT . getSizeInBits ( ) == <NUM_LIT> ) N = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Ops ) ;


<s> SDValue <ISA_LIT>TargetLowering :: LowerVECTOR_SHIFT ( SDValue Op , SelectionDAG & DAG ) const { BuildVectorSDNode * BVN = <NUM_LIT> ; SDValue V1 = Op . getOperand ( <NUM_LIT> ) ; SDValue V2 = Op . getOperand ( <NUM_LIT> ) ; SDValue V3 ; SDLoc dl ( Op ) ; EVT VT = Op . getValueType ( ) ; if ( ( BVN = dyn_cast < BuildVectorSDNode > ( V1 . getNode ( ) ) ) && isCommonSplatElement ( BVN ) ) V3 = V2 ; else if ( ( BVN = dyn_cast < BuildVectorSDNode > ( V2 . getNode ( ) ) ) && isCommonSplatElement ( BVN ) ) V3 = V1 ; else return SDValue ( ) ; SDValue CommonSplat = BVN -> getOperand ( <NUM_LIT> ) ; SDValue Result ; if ( VT . getSimpleVT ( ) == <STR_LIT> ) { switch ( Op . getOpcode ( ) ) { case <STR_LIT> : Result = DAG . getNode ( <STR_LIT> , dl , VT , V3 , CommonSplat ) ; break ; case <STR_LIT> : Result = DAG . getNode ( <STR_LIT> , dl , VT , V3 , CommonSplat ) ; break ; case <STR_LIT> : Result = DAG . getNode ( <STR_LIT> , dl , VT , V3 , CommonSplat ) ; break ; default : return SDValue ( ) ; } } else if ( VT . getSimpleVT ( ) == <STR_LIT> ) { switch ( Op . getOpcode ( ) ) { case <STR_LIT> : Result = DAG . getNode ( <STR_LIT> , dl , VT , V3 , CommonSplat ) ; break ; case <STR_LIT> :


<s> size_t MaskLen = Mask . size ( ) ; int ElemSizeInBits = VT . getVectorElementType ( ) . getSizeInBits ( ) ; if ( ( Subtarget . useHVXSglOps ( ) && ( ElemSizeInBits * MaskLen ) == <NUM_LIT> * <NUM_LIT> ) || ( Subtarget . useHVXDblOps ( ) && ( ElemSizeInBits * MaskLen ) == <NUM_LIT> * <NUM_LIT> ) ) { StridedLoadKind Pattern = isStridedLoad ( Mask ) ; if ( Pattern == StridedLoadKind :: NoPattern ) return SDValue ( ) ; SDValue Vec0 = Op . getOperand ( <NUM_LIT> ) ; SDValue Vec1 = Op . getOperand ( <NUM_LIT> ) ; SDValue StridePattern = DAG . getConstant ( Pattern , dl , <STR_LIT> ) ; SDValue Ops [ ] = { Vec1 , Vec0 , StridePattern } ; return DAG . getNode ( <STR_LIT> , dl , VT , Ops ) ; } }


<s> static SDValue CreateCopyOfByValArgument ( SDValue Src , SDValue Dst , SDValue Chain , <STR_LIT> Flags , SelectionDAG & DAG , SDLoc dl ) {

SDValue SizeNode = DAG . getConstant ( Flags . getByValSize ( ) , <STR_LIT> ) ;


<s> return DAG . getMemcpy ( Chain , dl , Dst , Src , SizeNode , Flags . getByValAlign ( ) , false , false , false , MachinePointerInfo ( ) , MachinePointerInfo ( ) ) ;


<s> const char * <ISA_LIT>TargetLowering :: getTargetNodeName ( unsigned Opcode ) const { switch ( ( <STR_LIT> ) Opcode ) { case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ; case <STR_LIT> : return <STR_LIT> ;


<s> const SDLoc & dl ( Op ) ; unsigned Opc = Op . getOpcode ( ) ; if ( Opc == <STR_LIT> ) { SDValue P = Op . getOperand ( <NUM_LIT> ) ; switch ( P . getOpcode ( ) ) { case <STR_LIT> : return DCI . DAG . getConstant ( - <NUM_LIT> , dl , ty ( Op ) ) ; case <STR_LIT> : return getZero ( dl , ty ( Op ) , DCI . DAG ) ; default : break ; } } else if ( Opc == <STR_LIT> ) { SDValue Cond = Op . getOperand ( <NUM_LIT> ) ; if ( Cond -> getOpcode ( ) == <STR_LIT> ) { SDValue C0 = Cond . getOperand ( <NUM_LIT> ) , C1 = Cond . getOperand ( <NUM_LIT> ) ; if ( C1 -> getOpcode ( ) == <STR_LIT> ) { SDValue VSel = DCI . DAG . getNode ( <STR_LIT> , dl , ty ( Op ) , C0 , Op . getOperand ( <NUM_LIT> ) , Op . getOperand ( <NUM_LIT> ) ) ;


<s> case <STR_LIT> : if ( N -> getValueType ( <NUM_LIT> ) == <STR_LIT> ) { SDValue P = getInstr ( <STR_LIT> , dl , <STR_LIT> , N -> getOperand ( <NUM_LIT> ) , DAG ) ; SDValue T = DAG . getAnyExtOrTrunc ( P , dl , <STR_LIT> ) ; Results . push_back ( T ) ; } break ; }


<s> if ( ! Results . empty ( ) ) return ; } const SDLoc & dl ( N ) ; switch ( N -> getOpcode ( ) ) { case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return ; case <STR_LIT> :

if ( N -> getValueType ( <NUM_LIT> ) == <STR_LIT> ) {


<s> bool <ISA_LIT>TargetLowering :: allowsHvxMemoryAccess ( MVT VecTy , MachineMemOperand :: Flags Flags , bool * Fast ) const { if ( VecTy . getSizeInBits ( ) > <NUM_LIT> * Subtarget . getVectorLength ( ) ) return false ; if ( ! Subtarget . isHVXVectorType ( VecTy , false ) ) return false ;


<s> bool <ISA_LIT>TargetLowering :: allowsHvxMisalignedMemoryAccesses ( MVT VecTy , MachineMemOperand :: Flags Flags , bool * Fast ) const { if ( ! Subtarget . isHVXVectorType ( VecTy ) ) return false ; if ( Fast ) * Fast = true ;


<s> if ( ! Subtarget . isHVXVectorType ( VecTy ) ) return false ; if ( Fast ) * Fast = true ; return true ;


<s> } } else { for ( unsigned I = <NUM_LIT> ; I != VecLen ; I += <NUM_LIT> ) { unsigned B = <NUM_LIT> ; for ( ; B != <NUM_LIT> ; ++ B ) { if ( ! Values [ I + B ] . isUndef ( ) ) break ; } SDValue F = Values [ I + B ] ; AllT &= IsTrue ( F ) ; AllF &= IsFalse ( F ) ; SDValue Ext = ( B < <NUM_LIT> ) ? DAG . getZExtOrTrunc ( F , dl , <STR_LIT> ) : DAG . getUNDEF ( <STR_LIT> ) ; Bytes . push_back ( Ext ) ; for ( ; B != <NUM_LIT> ; ++ B ) assert ( Values [ I + B ] . isUndef ( ) || Values [ I + B ] == F ) ; } } if ( AllT ) return DAG . getNode ( <STR_LIT> , dl , VecTy ) ; if ( AllF ) return DAG . getNode ( <STR_LIT> , dl , VecTy ) ; MVT ByteTy = <STR_LIT> ( <STR_LIT> , HwLen ) ; SDValue ByteVec = buildHvxVectorReg ( Bytes , dl , ByteTy , DAG ) ;

return DAG . getNode ( <STR_LIT> , dl , VecTy , ByteVec ) ;


<s> SDValue Ext = ! V . isUndef ( ) ? DAG . getZExtOrTrunc ( V , dl , <STR_LIT> ) : DAG . getUNDEF ( <STR_LIT> ) ; for ( unsigned B = <NUM_LIT> ; B != BitBytes ; ++ B ) Bytes . push_back ( Ext ) ; } } else { for ( unsigned I = <NUM_LIT> ; I != VecLen ; I += <NUM_LIT> ) { unsigned B = <NUM_LIT> ; for ( ; B != <NUM_LIT> ; ++ B ) { if ( ! Values [ I + B ] . isUndef ( ) ) break ; } SDValue F = Values [ I + B ] ; AllT &= IsTrue ( F ) ; AllF &= IsFalse ( F ) ; SDValue Ext = ( B < <NUM_LIT> ) ? DAG . getZExtOrTrunc ( F , dl , <STR_LIT> ) : DAG . getUNDEF ( <STR_LIT> ) ; Bytes . push_back ( Ext ) ; for ( ; B != <NUM_LIT> ; ++ B ) assert ( Values [ I + B ] . isUndef ( ) || Values [ I + B ] == F ) ; } } if ( AllT ) return DAG . getNode ( <STR_LIT> , dl , VecTy ) ; if ( AllF ) return DAG . getNode ( <STR_LIT> , dl , VecTy ) ;

MVT ByteTy = <STR_LIT> ( <STR_LIT> , HwLen ) ;


<s> SDValue CP = LowerConstantPool ( DAG . getConstantPool ( CV , ByteTy , Alignment ) , DAG ) ; SDValue Bytes = DAG . getLoad ( ByteTy , dl , DAG . getEntryNode ( ) , CP , MachinePointerInfo :: getConstantPool ( MF ) , Alignment ) ; SDValue Sel = DAG . getSelect ( dl , VecTy , VecQ , DAG . getBitcast ( VecTy , Bytes ) , getZero ( dl , VecTy , DAG ) ) ; SDValue All1 = DAG . getSplatBuildVector ( <STR_LIT> , dl , DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ) ; SDValue Vrmpy = getInstr ( <STR_LIT> , dl , ByteTy , { Sel , All1 } , DAG ) ; SDValue Rot = getInstr ( <STR_LIT> , dl , ByteTy , { Vrmpy , Vrmpy , DAG . getTargetConstant ( <NUM_LIT> , dl , <STR_LIT> ) } , DAG ) ; SDValue Vor = DAG . getNode ( <STR_LIT> , dl , ByteTy , { Vrmpy , Rot } ) ;


<s> auto Hi32 = [ & DAG , & dl ] ( SDValue P ) { return DAG . getTargetExtractSubreg ( <STR_LIT> , dl , <STR_LIT> , P ) ; } ; SDValue W0 = isUndef ( PredV ) ? DAG . getUNDEF ( <STR_LIT> ) : DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , PredV ) ; Words [ IdxW ] . push_back ( Hi32 ( W0 ) ) ; Words [ IdxW ] . push_back ( Lo32 ( W0 ) ) ; while ( Bytes < BitBytes ) { IdxW ^= <NUM_LIT> ; Words [ IdxW ] . clear ( ) ; if ( Bytes < <NUM_LIT> ) { for ( const SDValue & W : Words [ IdxW ^ <NUM_LIT> ] ) { SDValue T = expandPredicate ( W , dl , DAG ) ; Words [ IdxW ] . push_back ( Hi32 ( T ) ) ; Words [ IdxW ] . push_back ( Lo32 ( T ) ) ; } } else { for ( const SDValue & W : Words [ IdxW ^ <NUM_LIT> ] ) { Words [ IdxW ] . push_back ( W ) ; Words [ IdxW ] . push_back ( W ) ; } } Bytes *= <NUM_LIT> ; } assert ( Bytes == BitBytes ) ; SDValue Vec = ZeroFill ? getZero ( dl , ByteTy , DAG ) : DAG . getUNDEF ( ByteTy ) ; SDValue S4 = DAG . getConstant ( HwLen - <NUM_LIT> , dl , <STR_LIT> ) ; for ( const SDValue & W : Words [ IdxW ] ) { Vec = DAG . getNode ( <STR_LIT> , dl , ByteTy , Vec , S4 ) ; Vec = DAG . getNode ( <STR_LIT> , dl , ByteTy , Vec , W ) ; }


<s> while ( Bytes < BitBytes ) { IdxW ^= <NUM_LIT> ; Words [ IdxW ] . clear ( ) ; if ( Bytes < <NUM_LIT> ) { for ( const SDValue & W : Words [ IdxW ^ <NUM_LIT> ] ) { SDValue T = expandPredicate ( W , dl , DAG ) ; Words [ IdxW ] . push_back ( Hi32 ( T ) ) ; Words [ IdxW ] . push_back ( Lo32 ( T ) ) ; } } else { for ( const SDValue & W : Words [ IdxW ^ <NUM_LIT> ] ) { Words [ IdxW ] . push_back ( W ) ; Words [ IdxW ] . push_back ( W ) ; } } Bytes *= <NUM_LIT> ; } assert ( Bytes == BitBytes ) ; SDValue Vec = ZeroFill ? getZero ( dl , ByteTy , DAG ) : DAG . getUNDEF ( ByteTy ) ; SDValue S4 = DAG . getConstant ( HwLen - <NUM_LIT> , dl , <STR_LIT> ) ; for ( const SDValue & W : Words [ IdxW ] ) { Vec = DAG . getNode ( <STR_LIT> , dl , ByteTy , Vec , S4 ) ; Vec = DAG . getNode ( <STR_LIT> , dl , ByteTy , Vec , W ) ;


<s> assert ( Subtarget . isHVXVectorType ( ResTy ) ) ; if ( ! ZeroExt ) return DAG . getNode ( <STR_LIT> , dl , ResTy , VecV ) ; assert ( ty ( VecV ) . getVectorNumElements ( ) == ResTy . getVectorNumElements ( ) ) ;

SDValue True = DAG . getNode ( <STR_LIT> , dl , ResTy , DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ) ;


<s> SDValue <ISA_LIT>TargetLowering :: extendHvxVectorPred ( SDValue VecV , const SDLoc & dl , MVT ResTy , bool ZeroExt , SelectionDAG & DAG ) const { assert ( Subtarget . isHVXVectorType ( ResTy ) ) ; if ( ! ZeroExt ) return DAG . getNode ( <STR_LIT> , dl , ResTy , VecV ) ; assert ( ty ( VecV ) . getVectorNumElements ( ) == ResTy . getVectorNumElements ( ) ) ; SDValue True = DAG . getNode ( <STR_LIT> , dl , ResTy , DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ) ;

SDValue False = getZero ( dl , ResTy , DAG ) ;


<s> assert ( ResTy == <STR_LIT> ) ; unsigned HwLen = Subtarget . getVectorLength ( ) ; MVT ByteTy = <STR_LIT> ( <STR_LIT> , HwLen ) ; SDValue ByteVec = DAG . getNode ( <STR_LIT> , dl , ByteTy , VecV ) ;

unsigned Scale = HwLen / ty ( VecV ) . getVectorNumElements ( ) ;


<s> MVT ByteTy = <STR_LIT> ( <STR_LIT> , HwLen ) ; SDValue ByteVec = DAG . getNode ( <STR_LIT> , dl , ByteTy , VecV ) ; unsigned Scale = HwLen / ty ( VecV ) . getVectorNumElements ( ) ; SDValue ScV = DAG . getConstant ( Scale , dl , <STR_LIT> ) ; IdxV = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , IdxV , ScV ) ; SDValue ExtB = extractHvxElementReg ( ByteVec , IdxV , dl , <STR_LIT> , DAG ) ; SDValue Zero = DAG . getTargetConstant ( <NUM_LIT> , dl , <STR_LIT> ) ;

return getInstr ( <STR_LIT> , dl , <STR_LIT> , { ExtB , Zero } , DAG ) ;


<s> SDValue ByteIdx = convertToByteIndex ( IdxV , ElemTy , DAG ) ; SDValue ExWord = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , { VecV , ByteIdx } ) ; if ( ElemTy == <STR_LIT> ) return ExWord ; SDValue SubIdx = getIndexInWord32 ( IdxV , ElemTy , DAG ) ; SDValue ExVec = DAG . getBitcast ( tyVector ( ty ( ExWord ) , ElemTy ) , ExWord ) ;

return extractVector ( ExVec , SubIdx , dl , ElemTy , <STR_LIT> , DAG ) ;


<s> SDValue SubIdx = getIndexInWord32 ( IdxV , ElemTy , DAG ) ; SDValue ExVec = DAG . getBitcast ( tyVector ( ty ( ExWord ) , ElemTy ) , ExWord ) ; return extractVector ( ExVec , SubIdx , dl , ElemTy , <STR_LIT> , DAG ) ;


<s> for ( unsigned i = <NUM_LIT> ; i != HwLen / Rep ; ++ i ) { for ( unsigned j = <NUM_LIT> ; j != Rep ; ++ j ) Mask . push_back ( i + Offset ) ; } SDValue ShuffV = DAG . getVectorShuffle ( ByteTy , dl , ByteVec , Undef , Mask ) ; return DAG . getNode ( <STR_LIT> , dl , ResTy , ShuffV ) ; } unsigned Rep = <NUM_LIT> / ResLen ; for ( unsigned r = <NUM_LIT> ; r != HwLen / ResLen ; ++ r ) { for ( unsigned i = <NUM_LIT> ; i != ResLen ; ++ i ) { for ( unsigned j = <NUM_LIT> ; j != Rep ; ++ j ) Mask . push_back ( Offset + i * BitBytes ) ; } } SDValue Zero = getZero ( dl , <STR_LIT> , DAG ) ; SDValue ShuffV = DAG . getVectorShuffle ( ByteTy , dl , ByteVec , Undef , Mask ) ;

SDValue W0 = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , { ShuffV , Zero } ) ;


<s> unsigned WordIdx = ( Idx * ElemWidth ) / <NUM_LIT> ; SDValue W0Idx = DAG . getConstant ( WordIdx , dl , <STR_LIT> ) ; SDValue W0 = extractHvxElementReg ( WordVec , W0Idx , dl , <STR_LIT> , DAG ) ; if ( ResTy . getSizeInBits ( ) == <NUM_LIT> ) return DAG . getBitcast ( ResTy , W0 ) ; SDValue W1Idx = DAG . getConstant ( WordIdx + <NUM_LIT> , dl , <STR_LIT> ) ; SDValue W1 = extractHvxElementReg ( WordVec , W1Idx , dl , <STR_LIT> , DAG ) ; SDValue WW = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , { W1 , W0 } ) ;

return DAG . getBitcast ( ResTy , WW ) ;


<s> MVT OpTy = ty ( Op0 ) ; assert ( OpTy == ty ( Op1 ) ) ; MVT ElemTy = OpTy . getVectorElementType ( ) ; if ( ElemTy == <STR_LIT> ) return DAG . getVectorShuffle ( OpTy , dl , Op0 , Op1 , Mask ) ; assert ( ElemTy . getSizeInBits ( ) >= <NUM_LIT> ) ; MVT ResTy = tyVector ( OpTy , <STR_LIT> ) ; unsigned ElemSize = ElemTy . getSizeInBits ( ) / <NUM_LIT> ; SmallVector < int , <NUM_LIT> > ByteMask ; for ( int M : Mask ) { if ( M < <NUM_LIT> ) { for ( unsigned I = <NUM_LIT> ; I != ElemSize ; ++ I ) ByteMask . push_back ( - <NUM_LIT> ) ; } else { int NewM = M * ElemSize ; for ( unsigned I = <NUM_LIT> ; I != ElemSize ; ++ I ) ByteMask . push_back ( NewM + I ) ;


<s> SDValue <ISA_LIT>TargetLowering :: getIndexInWord32 ( SDValue Idx , MVT ElemTy , SelectionDAG & DAG ) const { unsigned ElemWidth = ElemTy . getSizeInBits ( ) ; assert ( ElemWidth >= <NUM_LIT> && ElemWidth <= <NUM_LIT> ) ; if ( ElemWidth == <NUM_LIT> ) return Idx ; if ( ty ( Idx ) != <STR_LIT> ) Idx = DAG . getBitcast ( <STR_LIT> , Idx ) ; const SDLoc & dl ( Idx ) ; SDValue Mask = DAG . getConstant ( <NUM_LIT> / ElemWidth - <NUM_LIT> , dl , <STR_LIT> ) ; SDValue SubIdx = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , { Idx , Mask } ) ; return SubIdx ;


<s> SDValue Mask = DAG . getConstant ( <NUM_LIT> / ElemWidth - <NUM_LIT> , dl , <STR_LIT> ) ; SDValue SubIdx = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , { Idx , Mask } ) ; return SubIdx ;


<s> SmallVector < SDValue , <NUM_LIT> > IntOps ; IntOps . push_back ( DAG . getConstant ( IntId , dl , <STR_LIT> ) ) ;


<s> if ( ElemTy == <STR_LIT> ) { for ( MVT T : Tys ) { assert ( T != <STR_LIT> ) ; auto A = getPreferredHvxVectorAction ( <STR_LIT> ( T , VecLen ) ) ; if ( A != ~ <NUM_LIT> ) return A ; } return ~ <NUM_LIT> ; }

if ( llvm :: is_contained ( Tys , ElemTy ) ) {


<s> if ( A != ~ <NUM_LIT> ) return A ; } return ~ <NUM_LIT> ; } if ( llvm :: is_contained ( Tys , ElemTy ) ) { unsigned VecWidth = VecTy . getSizeInBits ( ) ; bool HaveThreshold = HvxWidenThreshold . getNumOccurrences ( ) > <NUM_LIT> ; if ( HaveThreshold && <NUM_LIT> * HvxWidenThreshold <= VecWidth ) return TargetLoweringBase :: TypeWidenVector ; unsigned HwWidth = <NUM_LIT> * HwLen ; if ( VecWidth >= HwWidth / <NUM_LIT> && VecWidth < HwWidth ) return TargetLoweringBase :: TypeWidenVector ; } return ~ <NUM_LIT> ;


<s> unsigned HwLen = Subtarget . getVectorLength ( ) ; MVT ByteTy = <STR_LIT> ( <STR_LIT> , HwLen ) ; SDValue ByteVec = DAG . getNode ( <STR_LIT> , dl , ByteTy , VecV ) ; unsigned Scale = HwLen / ty ( VecV ) . getVectorNumElements ( ) ; SDValue ScV = DAG . getConstant ( Scale , dl , <STR_LIT> ) ; IdxV = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , IdxV , ScV ) ; ValV = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , ValV ) ; SDValue InsV = insertHvxElementReg ( ByteVec , IdxV , ValV , dl , DAG ) ;

return DAG . getNode ( <STR_LIT> , dl , ty ( VecV ) , InsV ) ;


<s> IdxV = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , IdxV , ScV ) ; ValV = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , ValV ) ; SDValue InsV = insertHvxElementReg ( ByteVec , IdxV , ValV , dl , DAG ) ; return DAG . getNode ( <STR_LIT> , dl , ty ( VecV ) , InsV ) ;


<s> assert ( Subtarget . isHVXVectorType ( VecTy , true ) ) ; unsigned VecLen = VecTy . getVectorNumElements ( ) ; unsigned HwLen = Subtarget . getVectorLength ( ) ; assert ( HwLen % VecLen == <NUM_LIT> && <STR_LIT> ) ; unsigned Scale = VecLen / SubTy . getVectorNumElements ( ) ; unsigned BitBytes = HwLen / VecLen ; unsigned BlockLen = HwLen / Scale ; MVT ByteTy = <STR_LIT> ( <STR_LIT> , HwLen ) ; SDValue ByteVec = DAG . getNode ( <STR_LIT> , dl , ByteTy , VecV ) ; SDValue ByteSub = createHvxPrefixPred ( SubV , dl , BitBytes , false , DAG ) ; SDValue ByteIdx ; auto * IdxN = dyn_cast < ConstantSDNode > ( IdxV . getNode ( ) ) ; if ( ! IdxN || ! IdxN -> isZero ( ) ) { ByteIdx = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , IdxV , DAG . getConstant ( BitBytes , dl , <STR_LIT> ) ) ; ByteVec = DAG . getNode ( <STR_LIT> , dl , ByteTy , ByteVec , ByteIdx ) ; }

assert ( BlockLen < HwLen && <STR_LIT> ) ;


<s> V1 = DAG . getTargetExtractSubreg ( <STR_LIT> , dl , SingleTy , VecV ) ; SDValue HalfV = DAG . getConstant ( SingleTy . getVectorNumElements ( ) , dl , <STR_LIT> ) ; PickHi = DAG . getSetCC ( dl , <STR_LIT> , IdxV , HalfV , <STR_LIT> ) ; if ( isHvxSingleTy ( SubTy ) ) { if ( const auto * CN = dyn_cast < const ConstantSDNode > ( IdxV . getNode ( ) ) ) { unsigned Idx = CN -> getZExtValue ( ) ; assert ( Idx == <NUM_LIT> || Idx == VecTy . getVectorNumElements ( ) / <NUM_LIT> ) ; unsigned SubIdx = ( Idx == <NUM_LIT> ) ? <STR_LIT> : <STR_LIT> ; return DAG . getTargetInsertSubreg ( SubIdx , dl , VecTy , VecV , SubV ) ; } SDValue InLo = DAG . getNode ( <STR_LIT> , dl , VecTy , { SubV , V1 } ) ; SDValue InHi = DAG . getNode ( <STR_LIT> , dl , VecTy , { V0 , SubV } ) ; return DAG . getNode ( <STR_LIT> , dl , VecTy , PickHi , InHi , InLo ) ; } SDValue S = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , IdxV , HalfV ) ; IdxV = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , PickHi , S , IdxV ) ; SingleV = DAG . getNode ( <STR_LIT> , dl , SingleTy , PickHi , V1 , V0 ) ; } assert ( SubTy . getSizeInBits ( ) == <NUM_LIT> || SubTy . getSizeInBits ( ) == <NUM_LIT> ) ; auto * IdxN = dyn_cast < ConstantSDNode > ( IdxV . getNode ( ) ) ; if ( ! IdxN || ! IdxN -> isZero ( ) ) { IdxV = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , IdxV , DAG . getConstant ( ElemWidth / <NUM_LIT> , dl , <STR_LIT> ) ) ; SingleV = DAG . getNode ( <STR_LIT> , dl , SingleTy , SingleV , IdxV ) ; } unsigned RolBase = HwLen ; if ( VecTy . getSizeInBits ( ) == <NUM_LIT> ) { SDValue V = DAG . getBitcast ( <STR_LIT> , SubV ) ; SingleV = DAG . getNode ( <STR_LIT> , dl , SingleTy , V ) ; } else { SDValue V = DAG . getBitcast ( <STR_LIT> , SubV ) ; SDValue R0 = DAG . getTargetExtractSubreg ( <STR_LIT> , dl , <STR_LIT> , V ) ; SDValue R1 = DAG . getTargetExtractSubreg ( <STR_LIT> , dl , <STR_LIT> , V ) ; SingleV = DAG . getNode ( <STR_LIT> , dl , SingleTy , SingleV , R0 ) ; SingleV = DAG . getNode ( <STR_LIT> , dl , SingleTy , SingleV , DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ) ;

SingleV = DAG . getNode ( <STR_LIT> , dl , SingleTy , SingleV , R1 ) ;


<s> bool <ISA_LIT>TargetLowering :: isHvxBoolTy ( MVT Ty ) const { return Subtarget . isHVXVectorType ( Ty , true ) && Ty . getVectorElementType ( ) == <STR_LIT> ;


<s> bool <ISA_LIT>TargetLowering :: isHvxOperation ( SDNode * N , SelectionDAG & DAG ) const { if ( ! Subtarget . useHVXOps ( ) ) return false ; auto IsHvxTy = [ this ] ( EVT Ty ) { return Ty . isSimple ( ) && Subtarget . isHVXVectorType ( Ty . getSimpleVT ( ) , true ) ; } ; auto IsHvxOp = [ this ] ( SDValue Op ) { return Op . getValueType ( ) . isSimple ( ) && Subtarget . isHVXVectorType ( ty ( Op ) , true ) ; } ; if ( llvm :: any_of ( N -> values ( ) , IsHvxTy ) || llvm :: any_of ( N -> ops ( ) , IsHvxOp ) ) return true ; auto IsWidenedToHvx = [ this , & DAG ] ( SDValue Op ) { if ( ! Op . getValueType ( ) . isSimple ( ) ) return false ; MVT ValTy = ty ( Op ) ; return ValTy . isVector ( ) && shouldWidenToHvx ( ValTy , DAG ) ; } ; for ( int i = <NUM_LIT> , e = N -> getNumValues ( ) ; i != e ; ++ i ) { if ( IsWidenedToHvx ( SDValue ( N , i ) ) ) return true ; } return llvm :: any_of ( N -> ops ( ) , IsWidenedToHvx ) ;


<s> auto IsHvxTy = [ this ] ( EVT Ty ) { return Ty . isSimple ( ) && Subtarget . isHVXVectorType ( Ty . getSimpleVT ( ) , true ) ; } ; auto IsHvxOp = [ this ] ( SDValue Op ) { return Op . getValueType ( ) . isSimple ( ) && Subtarget . isHVXVectorType ( ty ( Op ) , true ) ; } ; if ( llvm :: any_of ( N -> values ( ) , IsHvxTy ) || llvm :: any_of ( N -> ops ( ) , IsHvxOp ) ) return true ; auto IsWidenedToHvx = [ this , & DAG ] ( SDValue Op ) { if ( ! Op . getValueType ( ) . isSimple ( ) ) return false ; MVT ValTy = ty ( Op ) ;

return ValTy . isVector ( ) && shouldWidenToHvx ( ValTy , DAG ) ;


<s> bool <ISA_LIT>TargetLowering :: isHvxPairTy ( MVT Ty ) const { return Subtarget . isHVXVectorType ( Ty ) && Ty . getSizeInBits ( ) == <NUM_LIT> * Subtarget . getVectorLength ( ) ;


<s> bool <ISA_LIT>TargetLowering :: isHvxPairTy ( MVT Ty ) const {

return Subtarget . isHVXVectorType ( Ty ) && Ty . getSizeInBits ( ) == <NUM_LIT> * Subtarget . getVectorLength ( ) ;


<s> return Subtarget . isHVXVectorType ( Ty ) && Ty . getSizeInBits ( ) == <NUM_LIT> * Subtarget . getVectorLength ( ) ;


<s> bool <ISA_LIT>TargetLowering :: isHvxSingleTy ( MVT Ty ) const { return Subtarget . isHVXVectorType ( Ty ) && Ty . getSizeInBits ( ) == <NUM_LIT> * Subtarget . getVectorLength ( ) ;


<s> MVT ElemTy = ty ( InpV ) . getVectorElementType ( ) ; if ( ElemTy == <STR_LIT> && Subtarget . isHVXVectorType ( ResTy ) ) return LowerHvxSignExt ( Op , DAG ) ; return DAG . getNode ( <STR_LIT> , SDLoc ( Op ) , ResTy , InpV ) ;


<s> SDValue VQ = compressHvxPred ( Val , dl , WordTy , DAG ) ; unsigned BitWidth = ResTy . getSizeInBits ( ) ; if ( BitWidth < <NUM_LIT> ) { SDValue W0 = extractHvxElementReg ( VQ , DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) , dl , <STR_LIT> , DAG ) ; if ( BitWidth == <NUM_LIT> ) return W0 ; assert ( BitWidth < <NUM_LIT> ) ; return DAG . getZExtOrTrunc ( W0 , dl , ResTy ) ; } assert ( BitWidth == <NUM_LIT> || BitWidth == <NUM_LIT> ) ; SmallVector < SDValue , <NUM_LIT> > Words ; for ( unsigned i = <NUM_LIT> ; i != BitWidth / <NUM_LIT> ; ++ i ) { SDValue W = extractHvxElementReg ( VQ , DAG . getConstant ( i , dl , <STR_LIT> ) , dl , <STR_LIT> , DAG ) ; Words . push_back ( W ) ; } SmallVector < SDValue , <NUM_LIT> > Combines ; assert ( Words . size ( ) % <NUM_LIT> == <NUM_LIT> ) ; for ( unsigned i = <NUM_LIT> , e = Words . size ( ) ; i < e ; i += <NUM_LIT> ) { SDValue C = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , { Words [ i + <NUM_LIT> ] , Words [ i ] } ) ; Combines . push_back ( C ) ; } if ( BitWidth == <NUM_LIT> ) return Combines [ <NUM_LIT> ] ; return DAG . getNode ( <STR_LIT> , dl , ResTy , Combines ) ; } if ( isHvxBoolTy ( ResTy ) && ValTy . isScalarInteger ( ) ) { unsigned BitWidth = ValTy . getSizeInBits ( ) ; unsigned HwLen = Subtarget . getVectorLength ( ) ; assert ( BitWidth == HwLen ) ; MVT ValAsVecTy = <STR_LIT> ( <STR_LIT> , BitWidth / <NUM_LIT> ) ; SDValue ValAsVec = DAG . getBitcast ( ValAsVecTy , Val ) ; SmallVector < SDValue , <NUM_LIT> > Bytes ; SmallVector < SDValue , <NUM_LIT> > Tmp ; for ( unsigned I = <NUM_LIT> ; I != HwLen / <NUM_LIT> ; ++ I ) { SDValue Idx = DAG . getConstant ( I , dl , <STR_LIT> ) ; SDValue Byte = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , ValAsVec , Idx ) ; for ( unsigned J = <NUM_LIT> ; J != <NUM_LIT> ; ++ J ) {


<s> SDValue <ISA_LIT>TargetLowering :: LowerHvxBuildVector ( SDValue Op , SelectionDAG & DAG ) const { const SDLoc & dl ( Op ) ; MVT VecTy = ty ( Op ) ; unsigned Size = Op . getNumOperands ( ) ; SmallVector < SDValue , <NUM_LIT> > Ops ; for ( unsigned i = <NUM_LIT> ; i != Size ; ++ i ) Ops . push_back ( Op . getOperand ( i ) ) ; if ( VecTy . getSizeInBits ( ) == <NUM_LIT> * Subtarget . getVectorLength ( ) ) { ArrayRef < SDValue > A ( Ops ) ; MVT SingleTy = typeSplit ( VecTy ) . first ; SDValue V0 = buildHvxVectorReg ( A . take_front ( Size / <NUM_LIT> ) , dl , SingleTy , DAG ) ; SDValue V1 = buildHvxVectorReg ( A . drop_front ( Size / <NUM_LIT> ) , dl , SingleTy , DAG ) ; return DAG . getNode ( <STR_LIT> , dl , VecTy , V0 , V1 ) ; } if ( VecTy . getVectorElementType ( ) == <STR_LIT> ) return buildHvxVectorPred ( Ops , dl , VecTy , DAG ) ; if ( VecTy . getVectorElementType ( ) == <STR_LIT> ) { SmallVector < SDValue , <NUM_LIT> > NewOps ; for ( unsigned i = <NUM_LIT> ; i != Size ; i ++ ) NewOps . push_back ( DAG . getBitcast ( <STR_LIT> , Ops [ i ] ) ) ; SDValue T0 = DAG . getNode ( <STR_LIT> , dl , tyVector ( VecTy , <STR_LIT> ) , NewOps ) ; return DAG . getBitcast ( tyVector ( VecTy , <STR_LIT> ) , T0 ) ; }

return buildHvxVectorReg ( Ops , dl , VecTy , DAG ) ;


<s> unsigned Size = Op . getNumOperands ( ) ; SmallVector < SDValue , <NUM_LIT> > Ops ; for ( unsigned i = <NUM_LIT> ; i != Size ; ++ i ) Ops . push_back ( Op . getOperand ( i ) ) ; if ( VecTy . getSizeInBits ( ) == <NUM_LIT> * Subtarget . getVectorLength ( ) ) { ArrayRef < SDValue > A ( Ops ) ; MVT SingleTy = typeSplit ( VecTy ) . first ; SDValue V0 = buildHvxVectorReg ( A . take_front ( Size / <NUM_LIT> ) , dl , SingleTy , DAG ) ; SDValue V1 = buildHvxVectorReg ( A . drop_front ( Size / <NUM_LIT> ) , dl , SingleTy , DAG ) ; return DAG . getNode ( <STR_LIT> , dl , VecTy , V0 , V1 ) ;


<s> case <STR_LIT> : Elems [ i ] = V . getOperand ( <NUM_LIT> ) ; break ; default : llvm_unreachable ( <STR_LIT> ) ; } } } return DAG . getBuildVector ( VecTy , dl , Elems ) ; } assert ( VecTy . getVectorElementType ( ) == <STR_LIT> ) ; unsigned HwLen = Subtarget . getVectorLength ( ) ; assert ( isPowerOf2_32 ( NumOp ) && HwLen % NumOp == <NUM_LIT> ) ; SDValue Op0 = Op . getOperand ( <NUM_LIT> ) ; if ( Subtarget . isHVXVectorType ( ty ( Op0 ) , true ) ) { if ( NumOp == <NUM_LIT> ) return DAG . getNode ( <STR_LIT> , dl , VecTy , Op0 , Op . getOperand ( <NUM_LIT> ) ) ; ArrayRef < SDUse > U ( Op . getNode ( ) -> ops ( ) ) ; SmallVector < SDValue , <NUM_LIT> > SV ( U . begin ( ) , U . end ( ) ) ; ArrayRef < SDValue > Ops ( SV ) ; MVT HalfTy = typeSplit ( VecTy ) . first ; SDValue V0 = DAG . getNode ( <STR_LIT> , dl , HalfTy , Ops . take_front ( NumOp / <NUM_LIT> ) ) ; SDValue V1 = DAG . getNode ( <STR_LIT> , dl , HalfTy , Ops . take_back ( NumOp / <NUM_LIT> ) ) ; return DAG . getNode ( <STR_LIT> , dl , VecTy , V0 , V1 ) ; } unsigned BitBytes = HwLen / VecTy . getVectorNumElements ( ) ; SmallVector < SDValue , <NUM_LIT> > Prefixes ; for ( SDValue V : Op . getNode ( ) -> op_values ( ) ) { SDValue P = createHvxPrefixPred ( V , dl , BitBytes , true , DAG ) ; Prefixes . push_back ( P ) ; } unsigned InpLen = ty ( Op . getOperand ( <NUM_LIT> ) ) . getVectorNumElements ( ) ; MVT ByteTy = <STR_LIT> ( <STR_LIT> , HwLen ) ;

SDValue S = DAG . getConstant ( InpLen * BitBytes , dl , <STR_LIT> ) ;


<s> SDValue InpV = Op . getOperand ( <NUM_LIT> ) ; assert ( ResTy == ty ( InpV ) ) ; MVT ElemTy = ty ( InpV ) . getVectorElementType ( ) ; unsigned ElemWidth = ElemTy . getSizeInBits ( ) ; SDValue Vec1 = DAG . getNode ( <STR_LIT> , dl , ResTy , DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ) ; SDValue VecW = DAG . getNode ( <STR_LIT> , dl , ResTy , DAG . getConstant ( ElemWidth , dl , <STR_LIT> ) ) ; SDValue VecN1 = DAG . getNode ( <STR_LIT> , dl , ResTy , DAG . getConstant ( - <NUM_LIT> , dl , <STR_LIT> ) ) ; SDValue A = DAG . getNode ( <STR_LIT> , dl , ResTy , { DAG . getNode ( <STR_LIT> , dl , ResTy , { InpV , VecN1 } ) , DAG . getNode ( <STR_LIT> , dl , ResTy , { InpV , Vec1 } ) } ) ;

return DAG . getNode ( <STR_LIT> , dl , ResTy , { VecW , DAG . getNode ( <STR_LIT> , dl , ResTy , A ) } ) ;


<s> assert ( Op . getOpcode ( ) == <STR_LIT> ) ;

return DAG . getNode ( <STR_LIT> , SDLoc ( Op ) , ty ( Op ) , Op . getOperand ( <NUM_LIT> ) ) ;


<s> SDValue <ISA_LIT>TargetLowering :: LowerHvxExtend ( SDValue Op , SelectionDAG & DAG ) const {

assert ( Op . getOpcode ( ) == <STR_LIT> ) ;


<s> if ( ElemTy == <STR_LIT> ) return extractHvxElementPred ( VecV , IdxV , dl , ty ( Op ) , DAG ) ;

return extractHvxElementReg ( VecV , IdxV , dl , ty ( Op ) , DAG ) ;


<s> SDValue <ISA_LIT>TargetLowering :: LowerHvxExtractElement ( SDValue Op , SelectionDAG & DAG ) const { SDValue VecV = Op . getOperand ( <NUM_LIT> ) ; MVT ElemTy = ty ( VecV ) . getVectorElementType ( ) ; const SDLoc & dl ( Op ) ; SDValue IdxV = Op . getOperand ( <NUM_LIT> ) ; if ( ElemTy == <STR_LIT> ) return extractHvxElementPred ( VecV , IdxV , dl , ty ( Op ) , DAG ) ; return extractHvxElementReg ( VecV , IdxV , dl , ty ( Op ) , DAG ) ;


<s> assert ( Idx % DstTy . getVectorNumElements ( ) == <NUM_LIT> ) ; ( void ) Idx ; const SDLoc & dl ( Op ) ; MVT ElemTy = SrcTy . getVectorElementType ( ) ;

if ( ElemTy == <STR_LIT> ) return extractHvxSubvectorPred ( SrcV , IdxV , dl , DstTy , DAG ) ;


<s> assert ( Idx % DstTy . getVectorNumElements ( ) == <NUM_LIT> ) ; ( void ) Idx ; const SDLoc & dl ( Op ) ; MVT ElemTy = SrcTy . getVectorElementType ( ) ; if ( ElemTy == <STR_LIT> ) return extractHvxSubvectorPred ( SrcV , IdxV , dl , DstTy , DAG ) ;

return extractHvxSubvectorReg ( SrcV , IdxV , dl , DstTy , DAG ) ;


<s> assert ( VecTy == <STR_LIT> && ArgTy == <STR_LIT> ) ; SDValue F16Vec = Op . getOperand ( <NUM_LIT> ) ; APFloat FloatVal = APFloat ( <NUM_LIT> ) ; bool Ignored ; FloatVal . convert ( APFloat :: IEEEhalf ( ) , APFloat :: rmNearestTiesToEven , & Ignored ) ; SDValue Fp16Ones = DAG . getConstantFP ( FloatVal , dl , ArgTy ) ; SDValue VmpyVec = getInstr ( <STR_LIT> , dl , VecTy , { F16Vec , Fp16Ones } , DAG ) ; MVT HalfTy = typeSplit ( VecTy ) . first ; VectorPair Pair = opSplit ( VmpyVec , dl , DAG ) ; SDValue LoVec = getInstr ( <STR_LIT> , dl , HalfTy , { Pair . first } , DAG ) ; SDValue HiVec = getInstr ( <STR_LIT> , dl , HalfTy , { Pair . second } , DAG ) ; SDValue ShuffVec = getInstr ( <STR_LIT> , dl , VecTy , { HiVec , LoVec , DAG . getConstant ( - <NUM_LIT> , dl , <STR_LIT> ) } , DAG ) ;


<s> assert ( Subtarget . useHVXQFloatOps ( ) ) ; assert ( Op -> getOpcode ( ) == <STR_LIT> ) ; MVT VecTy = ty ( Op ) ; MVT ArgTy = ty ( Op . getOperand ( <NUM_LIT> ) ) ; const SDLoc & dl ( Op ) ; assert ( VecTy == <STR_LIT> && ArgTy == <STR_LIT> ) ; SDValue F16Vec = Op . getOperand ( <NUM_LIT> ) ; APFloat FloatVal = APFloat ( <NUM_LIT> ) ; bool Ignored ; FloatVal . convert ( APFloat :: IEEEhalf ( ) , APFloat :: rmNearestTiesToEven , & Ignored ) ; SDValue Fp16Ones = DAG . getConstantFP ( FloatVal , dl , ArgTy ) ;

SDValue VmpyVec = getInstr ( <STR_LIT> , dl , VecTy , { F16Vec , Fp16Ones } , DAG ) ;


<s> SDValue T0 = DAG . getNode ( <STR_LIT> , dl , tyVector ( VecTy , <STR_LIT> ) , DAG . getBitcast ( tyVector ( VecTy , <STR_LIT> ) , VecV ) , DAG . getBitcast ( <STR_LIT> , ValV ) , IdxV ) ; return DAG . getBitcast ( tyVector ( VecTy , <STR_LIT> ) , T0 ) ;


<s> SDValue ValV = Op . getOperand ( <NUM_LIT> ) ; SDValue IdxV = Op . getOperand ( <NUM_LIT> ) ; const SDLoc & dl ( Op ) ; MVT VecTy = ty ( VecV ) ; MVT ElemTy = VecTy . getVectorElementType ( ) ;

if ( ElemTy == <STR_LIT> ) return insertHvxSubvectorPred ( VecV , ValV , IdxV , dl , DAG ) ;


<s> SDValue ValV = Op . getOperand ( <NUM_LIT> ) ; SDValue IdxV = Op . getOperand ( <NUM_LIT> ) ; const SDLoc & dl ( Op ) ; MVT VecTy = ty ( VecV ) ; MVT ElemTy = VecTy . getVectorElementType ( ) ; if ( ElemTy == <STR_LIT> ) return insertHvxSubvectorPred ( VecV , ValV , IdxV , dl , DAG ) ;

return insertHvxSubvectorReg ( VecV , ValV , IdxV , dl , DAG ) ;


<s> const SDLoc & dl ( Op ) ; MVT ResTy = ty ( Op ) ; unsigned IntNo = cast < ConstantSDNode > ( Op . getOperand ( <NUM_LIT> ) ) -> getZExtValue ( ) ; bool Use64b = Subtarget . useHVX64BOps ( ) ; unsigned IntPredCast = Use64b ? <STR_LIT> : <STR_LIT> ; if ( IntNo == IntPredCast ) { SDValue Vs = Op . getOperand ( <NUM_LIT> ) ; MVT OpTy = ty ( Vs ) ; if ( isHvxBoolTy ( ResTy ) && isHvxBoolTy ( OpTy ) ) { if ( ResTy == OpTy ) return Vs ; return DAG . getNode ( <STR_LIT> , dl , ResTy , Vs ) ; } } return Op ;


<s> SDValue Load = DAG . getLoad ( ValTy , dl , Chain , Base , MemOp ) ; SDValue Thru = cast < MaskedLoadSDNode > ( MaskN ) -> getPassThru ( ) ; if ( isUndef ( Thru ) ) return Load ; SDValue VSel = DAG . getNode ( <STR_LIT> , dl , ValTy , Mask , Load , Thru ) ; return DAG . getMergeValues ( { VSel , Load . getValue ( <NUM_LIT> ) } , dl ) ; } unsigned StoreOpc = <STR_LIT> ; SDValue Value = cast < MaskedStoreSDNode > ( MaskN ) -> getValue ( ) ; SDValue Offset0 = DAG . getTargetConstant ( <NUM_LIT> , dl , ty ( Base ) ) ; if ( MaskN -> getAlign ( ) . value ( ) % HwLen == <NUM_LIT> ) { SDValue Store = getInstr ( StoreOpc , dl , <STR_LIT> , { Mask , Base , Offset0 , Value , Chain } , DAG ) ; DAG . setNodeMemRefs ( cast < MachineSDNode > ( Store . getNode ( ) ) , { MemOp } ) ; return Store ; } auto StoreAlign = [ & ] ( SDValue V , SDValue A ) { SDValue Z = getZero ( dl , ty ( V ) , DAG ) ; SDValue LoV = getInstr ( <STR_LIT> , dl , ty ( V ) , { V , Z , A } , DAG ) ; SDValue HiV = getInstr ( <STR_LIT> , dl , ty ( V ) , { Z , V , A } , DAG ) ; return std :: make_pair ( LoV , HiV ) ; } ; MVT ByteTy = <STR_LIT> ( <STR_LIT> , HwLen ) ; MVT BoolTy = <STR_LIT> ( <STR_LIT> , HwLen ) ; SDValue MaskV = DAG . getNode ( <STR_LIT> , dl , ByteTy , Mask ) ; VectorPair Tmp = StoreAlign ( MaskV , Base ) ; VectorPair MaskU = { DAG . getNode ( <STR_LIT> , dl , BoolTy , Tmp . first ) , DAG . getNode ( <STR_LIT> , dl , BoolTy , Tmp . second ) } ; VectorPair ValueU = StoreAlign ( Value , Base ) ; SDValue Offset1 = DAG . getTargetConstant ( HwLen , dl , <STR_LIT> ) ; SDValue StoreLo = getInstr ( StoreOpc , dl , <STR_LIT> , { MaskU . first , Base , Offset0 , ValueU . first , Chain } , DAG ) ;

SDValue StoreHi = getInstr ( StoreOpc , dl , <STR_LIT> , { MaskU . second , Base , Offset1 , ValueU . second , Chain } , DAG ) ;


<s> break ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return SplitHvxPairOp ( Op , DAG ) ; } } switch ( Opc ) { default : break ; case <STR_LIT> : return LowerHvxBuildVector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxSplatVector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxConcatVectors ( Op , DAG ) ; case <STR_LIT> : return LowerHvxInsertSubvector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxInsertElement ( Op , DAG ) ; case <STR_LIT> : return LowerHvxExtractSubvector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxExtractElement ( Op , DAG ) ; case <STR_LIT> : return LowerHvxBitcast ( Op , DAG ) ; case <STR_LIT> : return LowerHvxAnyExt ( Op , DAG ) ; case <STR_LIT> : return LowerHvxSignExt ( Op , DAG ) ; case <STR_LIT> : return LowerHvxZeroExt ( Op , DAG ) ; case <STR_LIT> : return LowerHvxCttz ( Op , DAG ) ; case <STR_LIT> : return LowerHvxSelect ( Op , DAG ) ;


<s> case <STR_LIT> : case <STR_LIT> : if ( shouldWidenToHvx ( ty ( Op . getOperand ( <NUM_LIT> ) ) , DAG ) ) { if ( SDValue T = WidenHvxExtend ( Op , DAG ) ) Results . push_back ( T ) ; } break ; case <STR_LIT> : if ( shouldWidenToHvx ( ty ( Op . getOperand ( <NUM_LIT> ) ) , DAG ) ) { if ( SDValue T = WidenHvxSetCC ( Op , DAG ) ) Results . push_back ( T ) ; } break ; case <STR_LIT> : if ( shouldWidenToHvx ( ty ( Op . getOperand ( <NUM_LIT> ) ) , DAG ) ) { if ( SDValue T = WidenHvxTruncate ( Op , DAG ) ) Results . push_back ( T ) ; } break ; case <STR_LIT> : { if ( shouldWidenToHvx ( ty ( cast < StoreSDNode > ( N ) -> getValue ( ) ) , DAG ) ) { SDValue Store = WidenHvxStore ( Op , DAG ) ; Results . push_back ( Store ) ; } break ; } case <STR_LIT> : if ( isHvxPairTy ( ty ( Op ) ) ) { SDValue S = SplitHvxMemOp ( Op , DAG ) ; assert ( S -> getOpcode ( ) == <STR_LIT> ) ; Results . push_back ( S . getOperand ( <NUM_LIT> ) ) ; Results . push_back ( S . getOperand ( <NUM_LIT> ) ) ; } break ;


<s> unsigned ElemSize = HwLen / VecLen ; MVT VecTy = <STR_LIT> ( <STR_LIT> ( ElemSize * <NUM_LIT> ) , VecLen ) ; SDValue S = DAG . getNode ( <STR_LIT> , dl , VecTy , Op . getOperand ( <NUM_LIT> ) , DAG . getNode ( <STR_LIT> , dl , VecTy , Op . getOperand ( <NUM_LIT> ) ) , DAG . getNode ( <STR_LIT> , dl , VecTy , Op . getOperand ( <NUM_LIT> ) ) ) ;

return DAG . getNode ( <STR_LIT> , dl , ResTy , S ) ;


<s> SDValue <ISA_LIT>TargetLowering :: LowerHvxShift ( SDValue Op , SelectionDAG & DAG ) const { if ( SDValue S = getVectorShiftByInt ( Op , DAG ) ) return S ; return Op ;


<s> if ( SDValue S = getVectorShiftByInt ( Op , DAG ) ) return S ;


<s> if ( ArgTy == <STR_LIT> ) { MVT SplatTy = <STR_LIT> ( <STR_LIT> , VecTy . getVectorNumElements ( ) ) ; SDValue ToInt16 = DAG . getBitcast ( <STR_LIT> , Op . getOperand ( <NUM_LIT> ) ) ; SDValue ToInt32 = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , ToInt16 ) ; SDValue Splat = DAG . getNode ( <STR_LIT> , dl , SplatTy , ToInt32 ) ; return DAG . getBitcast ( VecTy , Splat ) ;


<s> SDValue <ISA_LIT>TargetLowering :: LowerHvxSplatVector ( SDValue Op , SelectionDAG & DAG ) const { const SDLoc & dl ( Op ) ; MVT VecTy = ty ( Op ) ; MVT ArgTy = ty ( Op . getOperand ( <NUM_LIT> ) ) ; if ( ArgTy == <STR_LIT> ) { MVT SplatTy = <STR_LIT> ( <STR_LIT> , VecTy . getVectorNumElements ( ) ) ; SDValue ToInt16 = DAG . getBitcast ( <STR_LIT> , Op . getOperand ( <NUM_LIT> ) ) ; SDValue ToInt32 = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , ToInt16 ) ; SDValue Splat = DAG . getNode ( <STR_LIT> , dl , SplatTy , ToInt32 ) ; return DAG . getBitcast ( VecTy , Splat ) ; }


<s> MVT ElemTy = ty ( InpV ) . getVectorElementType ( ) ; if ( ElemTy == <STR_LIT> && Subtarget . isHVXVectorType ( ResTy ) ) return extendHvxVectorPred ( InpV , SDLoc ( Op ) , ty ( Op ) , true , DAG ) ;


<s> MVT ElemTy = ty ( InpV ) . getVectorElementType ( ) ; if ( ElemTy == <STR_LIT> && Subtarget . isHVXVectorType ( ResTy ) ) return extendHvxVectorPred ( InpV , SDLoc ( Op ) , ty ( Op ) , true , DAG ) ; return Op ;


<s> SDValue <ISA_LIT>TargetLowering :: opCastElem ( SDValue Vec , MVT ElemTy , SelectionDAG & DAG ) const { if ( ty ( Vec ) . getVectorElementType ( ) == ElemTy ) return Vec ;

MVT CastTy = tyVector ( Vec . getValueType ( ) . getSimpleVT ( ) , ElemTy ) ;


<s> MVT CastTy = tyVector ( Vec . getValueType ( ) . getSimpleVT ( ) , ElemTy ) ; return DAG . getBitcast ( CastTy , Vec ) ;


<s> return DAG . getNode ( <STR_LIT> , dl , typeJoin ( ty ( Ops ) ) , Ops . second , Ops . first ) ;


<s> if ( Vec . getOpcode ( ) == <STR_LIT> ) return VectorPair ( Vec . getOperand ( <NUM_LIT> ) , Vec . getOperand ( <NUM_LIT> ) ) ; return DAG . SplitVector ( Vec , dl , Tys . first , Tys . second ) ;


<s> <ISA_LIT>TargetLowering :: VectorPair <ISA_LIT>TargetLowering :: opSplit ( SDValue Vec , const SDLoc & dl , SelectionDAG & DAG ) const { TypePair Tys = typeSplit ( ty ( Vec ) ) ;

if ( Vec . getOpcode ( ) == <STR_LIT> ) return VectorPair ( Vec . getOperand ( <NUM_LIT> ) , Vec . getOperand ( <NUM_LIT> ) ) ;


<s> case <STR_LIT> : { SDValue Cond = Ops [ <NUM_LIT> ] ; if ( Cond -> getOpcode ( ) == <STR_LIT> ) { SDValue C0 = Cond . getOperand ( <NUM_LIT> ) , C1 = Cond . getOperand ( <NUM_LIT> ) ; if ( C1 -> getOpcode ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , dl , ty ( Op ) , C0 , Ops [ <NUM_LIT> ] , Ops [ <NUM_LIT> ] ) ; } break ; } case <STR_LIT> : if ( Ops [ <NUM_LIT> ] . getOpcode ( ) == <STR_LIT> ) { if ( const auto * C = dyn_cast < ConstantSDNode > ( Ops [ <NUM_LIT> ] . getOperand ( <NUM_LIT> ) ) ) return C -> isZero ( ) ? DAG . getNode ( <STR_LIT> , dl , ty ( Op ) ) : DAG . getNode ( <STR_LIT> , dl , ty ( Op ) ) ; } break ; case <STR_LIT> : if ( Ops [ <NUM_LIT> ] . getOpcode ( ) == <STR_LIT> ) return DAG . getNode ( <STR_LIT> , dl , ty ( Op ) , DAG . getConstant ( - <NUM_LIT> , dl , <STR_LIT> ) ) ; if ( Ops [ <NUM_LIT> ] . getOpcode ( ) == <STR_LIT> ) return getZero ( dl , ty ( Op ) , DAG ) ; break ; case <STR_LIT> : if ( isUndef ( Ops [ <NUM_LIT> ] ) ) return Ops [ <NUM_LIT> ] ; ;


<s> if ( SDValue T = WidenHvxExtend ( Op , DAG ) ) Results . push_back ( T ) ; } break ; case <STR_LIT> : if ( shouldWidenToHvx ( ty ( Op ) , DAG ) ) { if ( SDValue T = WidenHvxSetCC ( Op , DAG ) ) Results . push_back ( T ) ; } break ; case <STR_LIT> : if ( shouldWidenToHvx ( ty ( Op ) , DAG ) ) {

if ( SDValue T = WidenHvxTruncate ( Op , DAG ) ) Results . push_back ( T ) ;


<s> assert ( WideTy . isSimple ( ) ) ; return Subtarget . isHVXVectorType ( WideTy . getSimpleVT ( ) , true ) ; }


<s> auto Action = getPreferredHvxVectorAction ( Ty ) ; if ( Action == TargetLoweringBase :: TypeWidenVector ) { EVT WideTy = getTypeToTransformTo ( * DAG . getContext ( ) , Ty ) ; assert ( WideTy . isSimple ( ) ) ; return Subtarget . isHVXVectorType ( WideTy . getSimpleVT ( ) , true ) ;


<s> if ( Op . getOpcode ( ) == <STR_LIT> ) { if ( const auto * N = dyn_cast < const VTSDNode > ( A . getNode ( ) ) ) P = SplitVTNode ( N ) ; } OpsL . push_back ( P . first ) ; OpsH . push_back ( P . second ) ; } MVT ResTy = ty ( Op ) ; MVT HalfTy = typeSplit ( ResTy ) . first ; SDValue L = DAG . getNode ( Op . getOpcode ( ) , dl , HalfTy , OpsL ) ; SDValue H = DAG . getNode ( Op . getOpcode ( ) , dl , HalfTy , OpsH ) ; SDValue S = DAG . getNode ( <STR_LIT> , dl , ResTy , L , H ) ;


<s> MVT NewElemTy = <STR_LIT> ( ElemTy . getSizeInBits ( ) * Factor ) ;

return <STR_LIT> ( NewElemTy , VecTy . getVectorNumElements ( ) ) ;


<s> MVT <ISA_LIT>TargetLowering :: typeJoin ( const TypePair & Tys ) const { assert ( Tys . first . getVectorElementType ( ) == Tys . second . getVectorElementType ( ) ) ; MVT ElemTy = Tys . first . getVectorElementType ( ) ; return <STR_LIT> ( ElemTy , Tys . first . getVectorNumElements ( ) + Tys . second . getVectorNumElements ( ) ) ;


<s> return <STR_LIT> ( ElemTy , Tys . first . getVectorNumElements ( ) + Tys . second . getVectorNumElements ( ) ) ;


<s> unsigned NumElem = VecTy . getVectorNumElements ( ) ; assert ( ( NumElem % <NUM_LIT> ) == <NUM_LIT> && <STR_LIT> ) ; MVT HalfTy = <STR_LIT> ( VecTy . getVectorElementType ( ) , NumElem / <NUM_LIT> ) ; return { HalfTy , HalfTy } ;


<s> MVT HalfTy = <STR_LIT> ( VecTy . getVectorElementType ( ) , NumElem / <NUM_LIT> ) ;


<s> MVT ElemTy = VecTy . getVectorElementType ( ) ; MVT NewElemTy = <STR_LIT> ( ElemTy . getSizeInBits ( ) / Factor ) ;


<s> MVT NewElemTy = <STR_LIT> ( ElemTy . getSizeInBits ( ) / Factor ) ; return <STR_LIT> ( NewElemTy , VecTy . getVectorNumElements ( ) ) ;


<s> const SDLoc & dl ( Op ) ; unsigned HwWidth = <NUM_LIT> * Subtarget . getVectorLength ( ) ; SDValue Op0 = Op . getOperand ( <NUM_LIT> ) ; MVT ResTy = ty ( Op ) ; MVT OpTy = ty ( Op0 ) ; if ( ! Subtarget . isHVXElementType ( OpTy ) || ! Subtarget . isHVXElementType ( ResTy ) ) return SDValue ( ) ; auto getFactor = [ HwWidth ] ( MVT Ty ) { unsigned Width = Ty . getSizeInBits ( ) ; return HwWidth > Width ? HwWidth / Width : <NUM_LIT> ; } ; auto getWideTy = [ getFactor ] ( MVT Ty ) { unsigned WideLen = Ty . getVectorNumElements ( ) * getFactor ( Ty ) ; return <STR_LIT> ( Ty . getVectorElementType ( ) , WideLen ) ; } ; unsigned Opcode = Op . getOpcode ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; SDValue WideOp = appendUndef ( Op0 , getWideTy ( OpTy ) , DAG ) ; SDValue WideRes = DAG . getNode ( Opcode , dl , getWideTy ( ResTy ) , WideOp ) ;


<s> SDValue <ISA_LIT>TargetLowering :: WidenHvxExtend ( SDValue Op , SelectionDAG & DAG ) const { const SDLoc & dl ( Op ) ; unsigned HwWidth = <NUM_LIT> * Subtarget . getVectorLength ( ) ; SDValue Op0 = Op . getOperand ( <NUM_LIT> ) ; MVT ResTy = ty ( Op ) ; MVT OpTy = ty ( Op0 ) ; if ( ! Subtarget . isHVXElementType ( OpTy ) || ! Subtarget . isHVXElementType ( ResTy ) ) return SDValue ( ) ; auto getFactor = [ HwWidth ] ( MVT Ty ) { unsigned Width = Ty . getSizeInBits ( ) ; return HwWidth > Width ? HwWidth / Width : <NUM_LIT> ; } ; auto getWideTy = [ getFactor ] ( MVT Ty ) { unsigned WideLen = Ty . getVectorNumElements ( ) * getFactor ( Ty ) ; return <STR_LIT> ( Ty . getVectorElementType ( ) , WideLen ) ; } ; unsigned Opcode = Op . getOpcode ( ) == <STR_LIT> ? <STR_LIT> : <STR_LIT> ; SDValue WideOp = appendUndef ( Op0 , getWideTy ( OpTy ) , DAG ) ;

SDValue WideRes = DAG . getNode ( Opcode , dl , getWideTy ( ResTy ) , WideOp ) ;


<s> MVT LoadTy = <STR_LIT> ( <STR_LIT> , HwLen ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; auto * MemOp = MF . getMachineMemOperand ( LoadN -> getMemOperand ( ) , <NUM_LIT> , HwLen ) ; SDValue Load = DAG . getMaskedLoad ( LoadTy , dl , Chain , Base , Offset , Mask , DAG . getUNDEF ( LoadTy ) , LoadTy , MemOp , <STR_LIT> , <STR_LIT> , false ) ; SDValue Value = opCastElem ( Load , ResTy . getVectorElementType ( ) , DAG ) ; return DAG . getMergeValues ( { Value , Chain } , dl ) ;


<s> const SDLoc & dl ( Op ) ; auto * LoadN = cast < LoadSDNode > ( Op . getNode ( ) ) ; assert ( LoadN -> isUnindexed ( ) && <STR_LIT> ) ; assert ( LoadN -> getMemoryVT ( ) . getVectorElementType ( ) != <STR_LIT> && <STR_LIT> ) ; SDValue Chain = LoadN -> getChain ( ) ; SDValue Base = LoadN -> getBasePtr ( ) ; SDValue Offset = DAG . getUNDEF ( <STR_LIT> ) ; MVT ResTy = ty ( Op ) ; unsigned HwLen = Subtarget . getVectorLength ( ) ; unsigned ResLen = ResTy . getStoreSize ( ) ; assert ( ResLen < HwLen && <STR_LIT> ) ; MVT BoolTy = <STR_LIT> ( <STR_LIT> , HwLen ) ; SDValue Mask = getInstr ( <STR_LIT> , dl , BoolTy , { DAG . getConstant ( ResLen , dl , <STR_LIT> ) } , DAG ) ; MVT LoadTy = <STR_LIT> ( <STR_LIT> , HwLen ) ; MachineFunction & MF = DAG . getMachineFunction ( ) ; auto * MemOp = MF . getMachineMemOperand ( LoadN -> getMemOperand ( ) , <NUM_LIT> , HwLen ) ; SDValue Load = DAG . getMaskedLoad ( LoadTy , dl , Chain , Base , Offset , Mask , DAG . getUNDEF ( LoadTy ) , LoadTy , MemOp , <STR_LIT> , <STR_LIT> , false ) ;

SDValue Value = opCastElem ( Load , ResTy . getVectorElementType ( ) , DAG ) ;


<s> unsigned HwLen = Subtarget . getVectorLength ( ) ; unsigned WideOpLen = ( <NUM_LIT> * HwLen ) / ElemTy . getSizeInBits ( ) ; assert ( WideOpLen * ElemTy . getSizeInBits ( ) == <NUM_LIT> * HwLen ) ; MVT WideOpTy = <STR_LIT> ( ElemTy , WideOpLen ) ; if ( ! Subtarget . isHVXVectorType ( WideOpTy , true ) ) return SDValue ( ) ; SDValue WideOp0 = appendUndef ( Op0 , WideOpTy , DAG ) ; SDValue WideOp1 = appendUndef ( Op1 , WideOpTy , DAG ) ; EVT ResTy = getSetCCResultType ( DAG . getDataLayout ( ) , * DAG . getContext ( ) , WideOpTy ) ; SDValue SetCC = DAG . getNode ( <STR_LIT> , dl , ResTy , { WideOp0 , WideOp1 , Op . getOperand ( <NUM_LIT> ) } ) ; EVT RetTy = getTypeToTransformTo ( * DAG . getContext ( ) , ty ( Op ) ) ;

return DAG . getNode ( <STR_LIT> , dl , RetTy , { SetCC , getZero ( dl , <STR_LIT> , DAG ) } ) ;


<s> auto * StoreN = cast < StoreSDNode > ( Op . getNode ( ) ) ; assert ( StoreN -> isUnindexed ( ) && <STR_LIT> ) ; assert ( StoreN -> getMemoryVT ( ) . getVectorElementType ( ) != <STR_LIT> && <STR_LIT> ) ; SDValue Chain = StoreN -> getChain ( ) ; SDValue Base = StoreN -> getBasePtr ( ) ; SDValue Offset = DAG . getUNDEF ( <STR_LIT> ) ; SDValue Value = opCastElem ( StoreN -> getValue ( ) , <STR_LIT> , DAG ) ; MVT ValueTy = ty ( Value ) ;

unsigned ValueLen = ValueTy . getVectorNumElements ( ) ;


<s> } ; auto getWideTy = [ getFactor ] ( MVT Ty ) { unsigned WideLen = Ty . getVectorNumElements ( ) * getFactor ( Ty ) ; return <STR_LIT> ( Ty . getVectorElementType ( ) , WideLen ) ; } ; if ( Subtarget . isHVXVectorType ( OpTy ) ) return DAG . getNode ( <STR_LIT> , dl , getWideTy ( ResTy ) , Op0 ) ; assert ( ! isTypeLegal ( OpTy ) && <STR_LIT> ) ;

SDValue WideOp = appendUndef ( Op0 , getWideTy ( OpTy ) , DAG ) ;


<s> const SDLoc & dl ( Op ) ; unsigned HwWidth = <NUM_LIT> * Subtarget . getVectorLength ( ) ; SDValue Op0 = Op . getOperand ( <NUM_LIT> ) ; MVT ResTy = ty ( Op ) ; MVT OpTy = ty ( Op0 ) ; if ( ! Subtarget . isHVXElementType ( OpTy ) || ! Subtarget . isHVXElementType ( ResTy ) ) return SDValue ( ) ; auto getFactor = [ HwWidth ] ( MVT Ty ) { unsigned Width = Ty . getSizeInBits ( ) ; assert ( HwWidth % Width == <NUM_LIT> ) ; return HwWidth / Width ; } ; auto getWideTy = [ getFactor ] ( MVT Ty ) { unsigned WideLen = Ty . getVectorNumElements ( ) * getFactor ( Ty ) ; return <STR_LIT> ( Ty . getVectorElementType ( ) , WideLen ) ; } ; if ( Subtarget . isHVXVectorType ( OpTy ) ) return DAG . getNode ( <STR_LIT> , dl , getWideTy ( ResTy ) , Op0 ) ;

assert ( ! isTypeLegal ( OpTy ) && <STR_LIT> ) ;


<s> return DAG . getLoad ( VecTy , dl , DAG . getEntryNode ( ) , CP , MachinePointerInfo :: getConstantPool ( MF ) , Alignment ) ; } auto IsBuildFromExtracts = [ this , & Values ] ( SDValue & SrcVec , SmallVectorImpl < int > & SrcIdx ) { SDValue Vec ; for ( SDValue V : Values ) { if ( isUndef ( V ) ) { SrcIdx . push_back ( - <NUM_LIT> ) ; continue ; } if ( V . getOpcode ( ) != <STR_LIT> ) return false ; SDValue T = V . getOperand ( <NUM_LIT> ) ; if ( Vec . getNode ( ) != nullptr && T . getNode ( ) != Vec . getNode ( ) ) return false ; Vec = T ; ConstantSDNode * C = dyn_cast < ConstantSDNode > ( V . getOperand ( <NUM_LIT> ) ) ; if ( C == nullptr ) return false ; int I = C -> getSExtValue ( ) ; assert ( I >= <NUM_LIT> && <STR_LIT> ) ; SrcIdx . push_back ( I ) ; } SrcVec = Vec ; return true ; } ; SmallVector < int , <NUM_LIT> > ExtIdx ; SDValue ExtVec ; if ( IsBuildFromExtracts ( ExtVec , ExtIdx ) ) { MVT ExtTy = ty ( ExtVec ) ; unsigned ExtLen = ExtTy . getVectorNumElements ( ) ; if ( ExtLen == VecLen || ExtLen == <NUM_LIT> * VecLen ) { SmallVector < int , <NUM_LIT> > Mask ; BitVector Used ( ExtLen ) ; for ( int M : ExtIdx ) { Mask . push_back ( M ) ; if ( M >= <NUM_LIT> ) Used . set ( M ) ; } for ( unsigned I = <NUM_LIT> ; I != ExtLen ; ++ I ) { if ( Mask . size ( ) == ExtLen ) break ; if ( ! Used . test ( I ) ) Mask . push_back ( I ) ; } SDValue S = DAG . getVectorShuffle ( ExtTy , dl , ExtVec , DAG . getUNDEF ( ExtTy ) , Mask ) ; if ( ExtLen == VecLen ) return S ; return DAG . getTargetExtractSubreg ( <STR_LIT> , dl , VecTy , S ) ;


<s> MVT PartVT = <STR_LIT> ( VecTy . getVectorElementType ( ) , OpsPerWord ) ; for ( unsigned i = <NUM_LIT> ; i != VecLen ; i += OpsPerWord ) { SDValue W = buildVector32 ( Values . slice ( i , OpsPerWord ) , dl , PartVT , DAG ) ; Words . push_back ( DAG . getBitcast ( <STR_LIT> , W ) ) ; } } else { Words . assign ( Values . begin ( ) , Values . end ( ) ) ; } unsigned NumWords = Words . size ( ) ; bool IsSplat = true , IsUndef = true ; SDValue SplatV ; for ( unsigned i = <NUM_LIT> ; i != NumWords && IsSplat ; ++ i ) { if ( isUndef ( Words [ i ] ) ) continue ; IsUndef = false ; if ( ! SplatV . getNode ( ) ) SplatV = Words [ i ] ; else if ( SplatV != Words [ i ] ) IsSplat = false ; } if ( IsUndef ) return DAG . getUNDEF ( VecTy ) ; if ( IsSplat ) { assert ( SplatV . getNode ( ) ) ; auto * IdxN = dyn_cast < ConstantSDNode > ( SplatV . getNode ( ) ) ; if ( IdxN && IdxN -> isNullValue ( ) ) return getZero ( dl , VecTy , DAG ) ; return DAG . getNode ( <STR_LIT> , dl , VecTy , SplatV ) ; } SmallVector < ConstantInt * , <NUM_LIT> > Consts ( VecLen ) ; bool AllConst = getBuildVectorConstInts ( Values , VecTy , DAG , Consts ) ; if ( AllConst ) { ArrayRef < Constant * > Tmp ( ( Constant * * ) Consts . begin ( ) , ( Constant * * ) Consts . end ( ) ) ; Constant * CV = ConstantVector :: get ( Tmp ) ; unsigned Align = HwLen ; SDValue CP = LowerConstantPool ( DAG . getConstantPool ( CV , VecTy , Align ) , DAG ) ; return DAG . getLoad ( VecTy , dl , DAG . getEntryNode ( ) , CP , MachinePointerInfo :: getConstantPool ( MF ) , Align ) ; } assert ( <NUM_LIT> * Words . size ( ) == Subtarget . getVectorLength ( ) ) ; SDValue HalfV0 = getInstr ( <STR_LIT> , dl , VecTy , { } , DAG ) ; SDValue HalfV1 = getInstr ( <STR_LIT> , dl , VecTy , { } , DAG ) ; SDValue S = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ;

for ( unsigned i = <NUM_LIT> ; i != NumWords / <NUM_LIT> ; ++ i ) {


<s> if ( IsSplat ) { assert ( SplatV . getNode ( ) ) ; auto * IdxN = dyn_cast < ConstantSDNode > ( SplatV . getNode ( ) ) ; if ( IdxN && IdxN -> isNullValue ( ) ) return getZero ( dl , VecTy , DAG ) ; return DAG . getNode ( <STR_LIT> , dl , VecTy , SplatV ) ; } SmallVector < ConstantInt * , <NUM_LIT> > Consts ( VecLen ) ; bool AllConst = getBuildVectorConstInts ( Values , VecTy , DAG , Consts ) ; if ( AllConst ) { ArrayRef < Constant * > Tmp ( ( Constant * * ) Consts . begin ( ) , ( Constant * * ) Consts . end ( ) ) ; Constant * CV = ConstantVector :: get ( Tmp ) ; unsigned Align = HwLen ; SDValue CP = LowerConstantPool ( DAG . getConstantPool ( CV , VecTy , Align ) , DAG ) ; return DAG . getLoad ( VecTy , dl , DAG . getEntryNode ( ) , CP , MachinePointerInfo :: getConstantPool ( MF ) , Align ) ; } assert ( <NUM_LIT> * Words . size ( ) == Subtarget . getVectorLength ( ) ) ; SDValue HalfV0 = getInstr ( <STR_LIT> , dl , VecTy , { } , DAG ) ; SDValue HalfV1 = getInstr ( <STR_LIT> , dl , VecTy , { } , DAG ) ; SDValue S = DAG . getConstant ( <NUM_LIT> , dl , <STR_LIT> ) ; for ( unsigned i = <NUM_LIT> ; i != NumWords / <NUM_LIT> ; ++ i ) {

SDValue N = DAG . getNode ( <STR_LIT> , dl , VecTy , { HalfV0 , Words [ i ] } ) ;


<s> assert ( Op . getOpcode ( ) == <STR_LIT> ) ; return DAG . getZeroExtendVectorInReg ( Op . getOperand ( <NUM_LIT> ) , SDLoc ( Op ) , ty ( Op ) ) ;


<s> if ( IsPairOp ) { switch ( Opc ) { default : break ; case <STR_LIT> : case <STR_LIT> : return SplitHvxMemOp ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return SplitHvxPairOp ( Op , DAG ) ; } } switch ( Opc ) { default : break ; case <STR_LIT> : return LowerHvxBuildVector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxConcatVectors ( Op , DAG ) ; case <STR_LIT> : return LowerHvxInsertSubvector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxInsertElement ( Op , DAG ) ; case <STR_LIT> : return LowerHvxExtractSubvector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxExtractElement ( Op , DAG ) ; case <STR_LIT> : return LowerHvxAnyExt ( Op , DAG ) ; case <STR_LIT> : return LowerHvxSignExt ( Op , DAG ) ; case <STR_LIT> : return LowerHvxZeroExt ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return LowerHvxShift ( Op , DAG ) ; case <STR_LIT> : return LowerHvxMul ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : return LowerHvxMulh ( Op , DAG ) ; case <STR_LIT> : return LowerHvxExtend ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : return Op ; case <STR_LIT> : return SDValue ( ) ; } Op . dumpr ( & DAG ) ;


<s> } switch ( Opc ) { default : break ; case <STR_LIT> : return LowerHvxBuildVector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxConcatVectors ( Op , DAG ) ; case <STR_LIT> : return LowerHvxInsertSubvector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxInsertElement ( Op , DAG ) ; case <STR_LIT> : return LowerHvxExtractSubvector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxExtractElement ( Op , DAG ) ; case <STR_LIT> : return LowerHvxAnyExt ( Op , DAG ) ; case <STR_LIT> : return LowerHvxSignExt ( Op , DAG ) ; case <STR_LIT> : return LowerHvxZeroExt ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> :

case <STR_LIT> : return LowerHvxShift ( Op , DAG ) ;


<s> for ( SDValue V : Op . getNode ( ) -> ops ( ) ) DAG . ExtractVectorElements ( V , Elems ) ; return DAG . getBuildVector ( VecTy , dl , Elems ) ; } assert ( VecTy . getVectorElementType ( ) == <STR_LIT> ) ; unsigned HwLen = Subtarget . getVectorLength ( ) ; assert ( isPowerOf2_32 ( NumOp ) && HwLen % NumOp == <NUM_LIT> ) ; SDValue Op0 = Op . getOperand ( <NUM_LIT> ) ; if ( Subtarget . isHVXVectorType ( ty ( Op0 ) , true ) ) { if ( NumOp == <NUM_LIT> ) return DAG . getNode ( <STR_LIT> , dl , VecTy , Op0 , Op . getOperand ( <NUM_LIT> ) ) ; ArrayRef < SDUse > U ( Op . getNode ( ) -> ops ( ) ) ; SmallVector < SDValue , <NUM_LIT> > SV ( U . begin ( ) , U . end ( ) ) ; ArrayRef < SDValue > Ops ( SV ) ; MVT HalfTy = typeSplit ( VecTy ) . first ; SDValue V0 = DAG . getNode ( <STR_LIT> , dl , HalfTy , Ops . take_front ( NumOp / <NUM_LIT> ) ) ; SDValue V1 = DAG . getNode ( <STR_LIT> , dl , HalfTy , Ops . take_back ( NumOp / <NUM_LIT> ) ) ; return DAG . getNode ( <STR_LIT> , dl , VecTy , V0 , V1 ) ; } unsigned BitBytes = HwLen / VecTy . getVectorNumElements ( ) ; SmallVector < SDValue , <NUM_LIT> > Prefixes ; for ( SDValue V : Op . getNode ( ) -> op_values ( ) ) { SDValue P = createHvxPrefixPred ( V , dl , BitBytes , true , DAG ) ; Prefixes . push_back ( P ) ; } unsigned InpLen = ty ( Op . getOperand ( <NUM_LIT> ) ) . getVectorNumElements ( ) ; MVT ByteTy = <STR_LIT> ( <STR_LIT> , HwLen ) ; SDValue S = DAG . getConstant ( InpLen * BitBytes , dl , <STR_LIT> ) ; SDValue Res = getZero ( dl , ByteTy , DAG ) ; for ( unsigned i = <NUM_LIT> , e = Prefixes . size ( ) ; i != e ; ++ i ) { Res = DAG . getNode ( <STR_LIT> , dl , ByteTy , Res , S ) ; Res = DAG . getNode ( <STR_LIT> , dl , ByteTy , Res , Prefixes [ e - i - <NUM_LIT> ] ) ;


<s> } switch ( Opc ) { default : break ; case <STR_LIT> : return LowerHvxBuildVector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxConcatVectors ( Op , DAG ) ; case <STR_LIT> : return LowerHvxInsertSubvector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxInsertElement ( Op , DAG ) ; case <STR_LIT> : return LowerHvxExtractSubvector ( Op , DAG ) ; case <STR_LIT> : return LowerHvxExtractElement ( Op , DAG ) ; case <STR_LIT> : return LowerHvxUnalignedLoad ( Op , DAG ) ; case <STR_LIT> : return LowerHvxAnyExt ( Op , DAG ) ; case <STR_LIT> : return LowerHvxSignExt ( Op , DAG ) ; case <STR_LIT> : return LowerHvxZeroExt ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : case <STR_LIT> : return LowerHvxShift ( Op , DAG ) ; case <STR_LIT> : return LowerHvxMul ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : return LowerHvxMulh ( Op , DAG ) ; case <STR_LIT> : return LowerHvxExtend ( Op , DAG ) ; case <STR_LIT> : case <STR_LIT> : return Op ;


<s> auto BO = getBaseAndOffset ( Base ) ; unsigned BaseOpc = BO . first . getOpcode ( ) ; if ( BaseOpc == <STR_LIT> && BO . second % HwLen == <NUM_LIT> ) return Op ; const SDLoc & dl ( Op ) ; if ( BO . second % HwLen != <NUM_LIT> ) { BO . first = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , BO . first , DAG . getConstant ( BO . second % HwLen , dl , <STR_LIT> ) ) ; BO . second -= BO . second % HwLen ; } SDValue BaseNoOff = ( BaseOpc != <STR_LIT> ) ? DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , BO . first ) : BO . first ; SDValue Base0 = DAG . getMemBasePlusOffset ( BaseNoOff , BO . second , dl ) ; SDValue Base1 = DAG . getMemBasePlusOffset ( BaseNoOff , BO . second + HwLen , dl ) ; MachineMemOperand * WideMMO = nullptr ; if ( MachineMemOperand * MMO = LN -> getMemOperand ( ) ) { MachineFunction & MF = DAG . getMachineFunction ( ) ; WideMMO = MF . getMachineMemOperand ( MMO -> getPointerInfo ( ) , MMO -> getFlags ( ) , <NUM_LIT> * HwLen , HwLen , MMO -> getAAInfo ( ) , MMO -> getRanges ( ) , MMO -> getSyncScopeID ( ) , MMO -> getOrdering ( ) , MMO -> getFailureOrdering ( ) ) ; } SDValue Load0 = DAG . getLoad ( VecTy , dl , Chain , Base0 , WideMMO ) ; SDValue Load1 = DAG . getLoad ( VecTy , dl , Chain , Base1 , WideMMO ) ; SDValue Aligned = getInstr ( <STR_LIT> , dl , VecTy , { Load1 , Load0 , BaseNoOff . getOperand ( <NUM_LIT> ) } , DAG ) ; SDValue NewChain = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , Load0 . getValue ( <NUM_LIT> ) , Load1 . getValue ( <NUM_LIT> ) ) ; SDValue M = DAG . getMergeValues ( { Aligned , NewChain } , dl ) ; return M ;


<s> SDValue BaseNoOff = ( BaseOpc != <STR_LIT> ) ? DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , BO . first ) : BO . first ; SDValue Base0 = DAG . getMemBasePlusOffset ( BaseNoOff , BO . second , dl ) ; SDValue Base1 = DAG . getMemBasePlusOffset ( BaseNoOff , BO . second + HwLen , dl ) ; MachineMemOperand * WideMMO = nullptr ; if ( MachineMemOperand * MMO = LN -> getMemOperand ( ) ) { MachineFunction & MF = DAG . getMachineFunction ( ) ; WideMMO = MF . getMachineMemOperand ( MMO -> getPointerInfo ( ) , MMO -> getFlags ( ) , <NUM_LIT> * HwLen , HwLen , MMO -> getAAInfo ( ) , MMO -> getRanges ( ) , MMO -> getSyncScopeID ( ) , MMO -> getOrdering ( ) , MMO -> getFailureOrdering ( ) ) ; } SDValue Load0 = DAG . getLoad ( VecTy , dl , Chain , Base0 , WideMMO ) ; SDValue Load1 = DAG . getLoad ( VecTy , dl , Chain , Base1 , WideMMO ) ;

SDValue Aligned = getInstr ( <STR_LIT> , dl , VecTy , { Load1 , Load0 , BaseNoOff . getOperand ( <NUM_LIT> ) } , DAG ) ;


<s> if ( VecTy . getVectorElementType ( ) != <STR_LIT> ) { assert ( ( ElemSize == <NUM_LIT> || ElemSize == <NUM_LIT> ) && <STR_LIT> ) ; unsigned OpsPerWord = ( ElemSize == <NUM_LIT> ) ? <NUM_LIT> : <NUM_LIT> ; MVT PartVT = <STR_LIT> ( VecTy . getVectorElementType ( ) , OpsPerWord ) ; for ( unsigned i = <NUM_LIT> ; i != VecLen ; i += OpsPerWord ) { SDValue W = buildVector32 ( Values . slice ( i , OpsPerWord ) , dl , PartVT , DAG ) ; Words . push_back ( DAG . getBitcast ( <STR_LIT> , W ) ) ; } } else { Words . assign ( Values . begin ( ) , Values . end ( ) ) ; } unsigned NumWords = Words . size ( ) ; bool IsSplat = true , IsUndef = true ; SDValue SplatV ; for ( unsigned i = <NUM_LIT> ; i != NumWords && IsSplat ; ++ i ) { if ( isUndef ( Words [ i ] ) ) continue ; IsUndef = false ; if ( ! SplatV . getNode ( ) ) SplatV = Words [ i ] ; else if ( SplatV != Words [ i ] ) IsSplat = false ; } if ( IsUndef ) return DAG . getUNDEF ( VecTy ) ; if ( IsSplat ) { assert ( SplatV . getNode ( ) ) ; auto * IdxN = dyn_cast < ConstantSDNode > ( SplatV . getNode ( ) ) ; if ( IdxN && IdxN -> isNullValue ( ) ) return getZero ( dl , VecTy , DAG ) ; MVT WordTy = <STR_LIT> ( <STR_LIT> , HwLen / <NUM_LIT> ) ; SDValue S = DAG . getNode ( <STR_LIT> , dl , WordTy , SplatV ) ; return DAG . getBitcast ( VecTy , S ) ; } SmallVector < ConstantInt * , <NUM_LIT> > Consts ( VecLen ) ; bool AllConst = getBuildVectorConstInts ( Values , VecTy , DAG , Consts ) ; if ( AllConst ) { ArrayRef < Constant * > Tmp ( ( Constant * * ) Consts . begin ( ) , ( Constant * * ) Consts . end ( ) ) ; Constant * CV = ConstantVector :: get ( Tmp ) ; Align Alignment ( HwLen ) ; SDValue CP = LowerConstantPool ( DAG . getConstantPool ( CV , VecTy , Alignment ) , DAG ) ; return DAG . getLoad ( VecTy , dl , DAG . getEntryNode ( ) , CP , MachinePointerInfo :: getConstantPool ( MF ) , Alignment ) ; } auto IsBuildFromExtracts = [ this , & Values ] ( SDValue & SrcVec , SmallVectorImpl < int > & SrcIdx ) { SDValue Vec ; for ( SDValue V : Values ) { if ( isUndef ( V ) ) { SrcIdx . push_back ( - <NUM_LIT> ) ; continue ; } if ( V . getOpcode ( ) != <STR_LIT> ) return false ;

SDValue T = V . getOperand ( <NUM_LIT> ) ;


<s> MVT BoolTy = <STR_LIT> ( <STR_LIT> , HwLen ) ; SDValue Q = getInstr ( <STR_LIT> , dl , BoolTy , { DAG . getConstant ( BlockLen , dl , <STR_LIT> ) } , DAG ) ; ByteVec = getInstr ( <STR_LIT> , dl , ByteTy , { Q , ByteSub , ByteVec } , DAG ) ; if ( ! IdxN || ! IdxN -> isNullValue ( ) ) { SDValue HwLenV = DAG . getConstant ( HwLen , dl , <STR_LIT> ) ; SDValue ByteXdi = DAG . getNode ( <STR_LIT> , dl , <STR_LIT> , HwLenV , ByteIdx ) ; ByteVec = DAG . getNode ( <STR_LIT> , dl , ByteTy , ByteVec , ByteXdi ) ; } return DAG . getNode ( <STR_LIT> , dl , VecTy , ByteVec ) ;