text: /*
   * Copyright (C) 2010-2020 Arm Limited or its affiliates. All rights reserved.
   *
   * SPDX-License-Identifier: Apache-2.0
   *
   * Licensed under the Apache License, Version 2.0 (the License); you may
   * not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   * www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an AS IS BASIS, WITHOUT
   * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  
  /* ----------------------------------------------------------------------
   * Project:      CMSIS NN Library
   * Title:        arm_nnsupportfunctions.h
   * Description:  Public header file of support functions for CMSIS NN Library
   *
   * $Date:        09. October 2020
   * $Revision:    V.4.5.5
   *
   * Target Processor:  Cortex-M CPUs
   * -------------------------------------------------------------------- */
  
  #ifndef _ARM_NNSUPPORTFUNCTIONS_H_
  #define _ARM_NNSUPPORTFUNCTIONS_H_
  
  #include "arm_math_types.h"
  #include "arm_common_tables.h"
  
  #ifdef __cplusplus
  extern    "C"
  {
  #endif
  
  #define LEFT_SHIFT(_shift)  (_shift > 0 ? _shift : 0)
  #define RIGHT_SHIFT(_shift) (_shift > 0 ? 0 : -_shift)
  #define MASK_IF_ZERO(x)     (x) == 0 ? ~0 : 0
  #define MASK_IF_NON_ZERO(x) (x) != 0 ? ~0 : 0
  #define SELECT_USING_MASK(mask, a, b) ((mask) & (a)) ^ (~(mask) & (b))
  
  #define MAX(A,B) ((A) > (B) ? (A) : (B))
  #define MIN(A,B) ((A) < (B) ? (A) : (B))
  #define CLAMP(x, h, l) MAX(MIN((x), (h)), (l))
  
  /**
   * @brief Union for SIMD access of q31/q15/q7 types
   */
  union arm_nnword
  {
      q31_t     word;
                 /**< q31 type */
      q15_t     half_words[2];
                 /**< q15 type */
      q7_t      bytes[4];
                 /**< q7 type */
  };
  
  /**
   * @brief Union for data type long long
   */
  struct arm_nn_double
  {
    uint32_t low;
    int32_t high;
  };
  
  union arm_nn_long_long
  {
    int64_t long_long;
    struct arm_nn_double word;
  };
  
  /**
   * @defgroup nndata_convert Neural Network Data Conversion Functions
   *
   * Perform data type conversion in-between neural network operations
   *
   */
  
  /**
   * @brief Converts the elements of the q7 vector to q15 vector without left-shift
   * @param[in]       *pSrc points to the q7 input vector
   * @param[out]      *pDst points to the q15 output vector
   * @param[in]       blockSize length of the input vector
   *
   */
  void arm_q7_to_q15_no_shift(const q7_t *pSrc, q15_t *pDst, uint32_t blockSize);
  
  /**
   * @brief Non-saturating addition of elements of a q7 vector
   * @param[in]       *input Pointer to the q7 input vector
   * @param[out]      *output Pointer to the q31 output variable.
   * @param[in]       block_size length of the input vector
   * \par Description:
   *
   * 2^24 samples can be added without saturating the result.
   *
   * The equation used for the conversion process is:
   *
   * <pre>
   *  sum = input[0] + input[1] + .. + input[block_size -1]
   * </pre>
   *
   * */
  void arm_nn_add_q7(const q7_t *input, q31_t *output, uint32_t block_size);
  
  /**
   * @brief  Converts the elements of the q7 vector to reordered q15 vector without left-shift
   * @param[in]       *pSrc points to the q7 input vector
   * @param[out]      *pDst points to the q15 output vector
   * @param[in]       blockSize length of the input vector
   * @return none.
   *
   */
  void arm_q7_to_q15_reordered_no_shift(const q7_t * pSrc, q15_t * pDst, uint32_t blockSize);
  
  /**
   * @brief Converts the elements from a q7 vector to a q15 vector with an added offset
   * @param[in]    src        pointer to the q7 input vector
   * @param[out]   dst        pointer to the q15 output vector
   * @param[in]    block_size length of the input vector
   * @param[in]    offset     q7 offset to be added to each input vector element.
   *
   * \par Description:
   *
   * The equation used for the conversion process is:
   *
   * <pre>
   *  dst[n] = (q15_t) src[n] + offset;   0 <= n < block_size.
   * </pre>
   *
   */
  void arm_q7_to_q15_with_offset(const q7_t *src, q15_t *dst, uint32_t block_size, q15_t offset);
  
  /**
   * @brief Converts the elements of the q7 vector to reordered q15 vector with an added offset
   * @param[in]       src        pointer to the q7 input vector
   * @param[out]      dst        pointer to the q15 output vector
   * @param[in]       block_size length of the input vector
   * @param[in]       offset     offset to be added to each input vector element.
   * @return none.
   *
   * @details  This function does the q7 to q15 expansion with re-ordering of bytes. Re-ordering is a consequence of
   *           the sign extension intrinsic(DSP extension). The tail (i.e., last (N % 4) elements) retains its original
   *           order.
   *
   */
  void arm_q7_to_q15_reordered_with_offset(const q7_t *src, q15_t *dst, uint32_t block_size, q15_t offset);
  
  /**
   * @brief Converts the elements from a q7 vector and accumulate to a q15 vector
   * @param[in]    *src       points to the q7 input vector
   * @param[out]   *dst       points to the q15 output vector
   * @param[in]    block_size length of the input vector
   *
   * \par Description:
   *
   * The equation used for the conversion process is:
   *
   * <pre>
   *  dst[n] += (q15_t) src[n] ;   0 <= n < block_size.
   * </pre>
   *
   */
  void arm_nn_accumulate_q7_to_q15(q15_t *dst, const q7_t *src, uint32_t block_size);
  
  /**
   * @brief Depthwise conv on an im2col buffer where the input channel equals output channel.
   * @param[in]    row     pointer to row
   * @param[in]    col     pointer to im2col buffer, always consists of 2 columns.
   * @param[in]    num_ch   number of channels
   * @param[in]    out_shift  pointer to per output channel requantization shift parameter.
   * @param[in]    out_mult   pointer to per output channel requantization multiplier parameter.
   * @param[in]    out_offset      output tensor offset.
   * @param[in]    activation_min   minimum value to clamp the output to. Range : int8
   * @param[in]    activation_max   maximum value to clamp the output to. Range : int8
   * @param[in]    kernel_size   number of elements in one column.
   * @param[in]    output_bias per output channel bias. Range : int32
   * @param[out]   out         pointer to output
   * @return     The function returns one of the two
   *              1. The incremented output pointer for a successful operation or
   *              2. NULL if implementation is not available.
   *
   * @details     Supported framework: TensorFlow Lite micro.
   */
  q7_t *arm_nn_depthwise_conv_s8_core(const q7_t *row,
                                      const q15_t *col,
                                      const uint16_t num_ch,
                                      const int32_t *out_shift,
                                      const int32_t *out_mult,
                                      const int32_t out_offset,
                                      const int32_t activation_min,
                                      const int32_t activation_max,
                                      const uint16_t kernel_size,
                                      const int32_t *const output_bias,
                                      q7_t *out);
  
  /**
   * @brief General Matrix-multiplication function with per-channel requantization.
   * @param[in]       input_row    pointer to row operand
   * @param[in]       input_col    pointer to col operand
   * @param[in]       output_ch    number of rows of input_row
   * @param[in]       col_batches  number of column batches. Range: 1 to 4
   * @param[in]       output_shift  pointer to per output channel requantization shift parameter.
   * @param[in]       output_mult   pointer to per output channel requantization multiplier parameter.
   * @param[in]       out_offset    output tensor offset.
   * @param[in]       col_offset    input tensor(col) offset.
   * @param[in]       row_offset    kernel offset(row). Not used.
   * @param[in]       out_activation_min   minimum value to clamp the output to. Range : int8
   * @param[in]       out_activation_max   maximum value to clamp the output to. Range : int8
   * @param[in]       row_len       number of elements in each row
   * @param[in]       bias          per output channel bias. Range : int32
   * @param[in,out]   out           pointer to output
   * @return     The function returns one of the two
   *              1. The incremented output pointer for a successful operation or
   *              2. NULL if implementation is not available.
   *
   * @details   Supported framework: TensorFlow Lite
  */
  q7_t *arm_nn_mat_mult_s8(const q7_t *input_row,
                           const q7_t *input_col,
                           const uint16_t output_ch,
                           const uint16_t col_batches,
                           const int32_t *output_shift,
                           const int32_t *output_mult,
                           const int32_t out_offset,
                           const int32_t col_offset,
                           const int32_t row_offset,
                           const int16_t out_activation_min,
                           const int16_t out_activation_max,
                           const uint16_t row_len,
                           const int32_t *const bias,
                           q7_t *out);
  
  /**
   * @brief General Matrix-multiplication without requantization for one row & one column
   * @param[in]       row_elements  number of row elements
   * @param[in]       row_base      pointer to row operand
   * @param[in]       col_base      pointer to col operand
   * @param[out]      sum_col       pointer to store sum of column elements
   * @param[out]      output        pointer to store result of multiply-accumulate
   * @return     The function returns the multiply-accumulated result of the row by column.
   *
   * @details Pseudo-code
   *      *output = 0
   *      sum_col = 0
   *      for (i = 0; i < row_elements; i++)
   *          *output += row_base[i] * col_base[i]
   *          sum_col += col_base[i]
   *
  */
  arm_status arm_nn_mat_mul_core_1x_s8(int32_t row_elements,
                                       const int8_t *row_base,
                                       const int8_t *col_base,
                                       int32_t *const sum_col,
                                       int32_t *const output);
  
  /**
   * @brief General Matrix-multiplication without requantization for four rows and one column
   * @param[in]       row_elements  number of row elements
   * @param[in]       offset        offset between rows. Can be the same as row_elements.
   *                                For e.g, in a 1x1 conv scenario with stride as 1.
   * @param[in]       row_base      pointer to row operand
   * @param[in]       col_base      pointer to col operand
   * @param[out]      sum_col       pointer to store sum of column elements
   * @param[out]      output        pointer to store result(4 int32's) of multiply-accumulate
   * @return     The function returns the multiply-accumulated result of the row by column
   *
   * @details Pseudo-code
   *      output[0] = 0
   *         ..
   *      output[3] = 0
   *      sum_col = 0
   *      for (i = 0; i < row_elements; i++)
   *          output[0] += row_base[i] * col_base[i]
   *                ..
   *          output[3] += row_base[i + (row_elements * 3)] * col_base[i]
   *          sum_col += col_base[i]
  */
  arm_status arm_nn_mat_mul_core_4x_s8(const int32_t row_elements,
                                       const int32_t offset,
                                       const int8_t *row_base,
                                       const int8_t *col_base,
                                       int32_t *const sum_col,
                                       int32_t *const output);
  
  /**
  * @brief General Matrix-multiplication function with per-channel requantization.
  *        This function assumes:
  *        - LHS input matrix NOT transposed (nt)
  *        - RHS input matrix transposed (t)
  *
  *  @note This operation also performs the broadcast bias addition before the requantization
  *
  * @param[in]  lhs                Pointer to the LHS input matrix
  * @param[in]  rhs                Pointer to the RHS input matrix
  * @param[in]  bias               Pointer to the bias vector. The length of this vector is equal to the number of output columns (or RHS input rows)
  * @param[out] dst                Pointer to the output matrix with "m" rows and "n" columns
  * @param[in]  dst_multipliers    Pointer to the multipliers vector needed for the per-channel requantization. The length of this vector is equal to
  *                                the number of output columns (or RHS input rows)
  * @param[in]  dst_shifts         Pointer to the shifts vector needed for the per-channel requantization. The length of this vector is equal to
  *                                the number of output columns (or RHS input rows)
  * @param[in]  lhs_rows           Number of LHS input rows
  * @param[in]  rhs_rows           Number of RHS input rows
  * @param[in]  rhs_cols           Number of LHS/RHS input columns
  * @param[in]  lhs_offset         Offset to be applied to the LHS input value
  * @param[in]  dst_offset         Offset to be applied the output result
  * @param[in]  activation_min     Minimum value to clamp down the output. Range : int8
  * @param[in]  activation_max     Maximum value to clamp up the output. Range : int8
  *
  * @return     The function returns <code>ARM_MATH_SUCCESS</code>
  *
  */
  arm_status arm_nn_mat_mult_nt_t_s8(const q7_t *lhs,
                                     const q7_t *rhs,
                                     const q31_t *bias,
                                     q7_t *dst,
                                     const int32_t *dst_multipliers,
                                     const int32_t *dst_shifts,
                                     const int32_t lhs_rows,
                                     const int32_t rhs_rows,
                                     const int32_t rhs_cols,
                                     const int32_t lhs_offset,
                                     const int32_t dst_offset,
                                     const int32_t activation_min,
                                     const int32_t activation_max);
  
  /**
   * @brief s8 Vector by Matrix (transposed) multiplication
   *
   * @param[in]      lhs             Input left-hand side vector
   * @param[in]      rhs             Input right-hand side matrix (transposed)
   * @param[in]      bias            Input bias
   * @param[out]     dst             Output vector
   * @param[in]      lhs_offset      Offset to be added to the input values of the left-hand side vector. Range: -127 to 128
   * @param[in]      rhs_offset      Offset to be added to the input values of the right-hand side matrix. Range: -127 to 128
   * @param[in]      dst_offset      Offset to be added to the output values. Range: -127 to 128
   * @param[in]      dst_multiplier  Output multiplier
   * @param[in]      dst_shift       Output shift
   * @param[in]      rhs_cols        Number of columns in the right-hand side input matrix
   * @param[in]      rhs_rows        Number of rows in the right-hand side input matrix
   * @param[in]      activation_min  Minimum value to clamp the output to. Range: int8
   * @param[in]      activation_max  Maximum value to clamp the output to. Range: int8
   *
   * @return         The function returns <code>ARM_MATH_SUCCESS</code>
   *
   */
  arm_status arm_nn_vec_mat_mult_t_s8(const q7_t *lhs,
                                      const q7_t *rhs,
                                      const q31_t *bias,
                                      q7_t *dst,
                                      const int32_t lhs_offset,
                                      const int32_t rhs_offset,
                                      const int32_t dst_offset,
                                      const int32_t dst_multiplier,
                                      const int32_t dst_shift,
                                      const int32_t rhs_cols,
                                      const int32_t rhs_rows,
                                      const int32_t activation_min,
                                      const int32_t activation_max);
  
  /**
   * @brief Depthwise convolution of transposed rhs matrix with 4 lhs matrices. To be used in padded cases where
   *        the padding is -lhs_offset(Range: int8). Dimensions are the same for lhs and rhs.
   *
   * @param[in]      lhs             Input left-hand side matrix
   * @param[in]      rhs             Input right-hand side matrix (transposed)
   * @param[in]      lhs_offset      LHS matrix offset(input offset). Range: -127 to 128
   * @param[in]      num_ch          Number of channels in LHS/RHS
   * @param[in]      out_shift       Per channel output shift. Length of vector is equal to number of channels
   * @param[in]      out_mult        Per channel output multiplier. Length of vector is equal to number of channels
   * @param[in]      out_offset      Offset to be added to the output values. Range: -127 to 128
   * @param[in]      activation_min  Minimum value to clamp the output to. Range: int8
   * @param[in]      activation_max  Maximum value to clamp the output to. Range: int8
   * @param[in]       row_x_col       (row_dimension * col_dimension) of LHS/RHS matrix
   * @param[in]      output_bias     Per channel output bias. Length of vector is equal to number of channels
   * @param[in]      out             Output pointer
   *
   * @return         The function returns one of the two
   *                  - Updated output pointer if an implementaiton is available
   *                  - NULL if no implementation is available.
   *
   * @note           If number of channels is not a multiple of 4, upto 3 elements outside the boundary will be read out
   *                 for the following.
   *                  - Output shift
   *                  - Output multiplier
   *                  - Output bias
   *                  - rhs
   */
  q7_t *arm_nn_depthwise_conv_nt_t_padded_s8(const q7_t *lhs,
                                             const q7_t *rhs,
                                             const int32_t lhs_offset,
                                             const uint16_t num_ch,
                                             const int32_t *out_shift,
                                             const int32_t *out_mult,
                                             const int32_t out_offset,
                                             const int32_t activation_min,
                                             const int32_t activation_max,
                                             const uint16_t row_x_col,
                                             const int32_t *const output_bias,
                                             q7_t *out);
  
  /**
   * @brief Depthwise convolution of transposed rhs matrix with 4 lhs matrices. To be used in non-padded cases.
   *        Dimensions are the same for lhs and rhs.
   *
   * @param[in]      lhs             Input left-hand side matrix
   * @param[in]      rhs             Input right-hand side matrix (transposed)
   * @param[in]      lhs_offset      LHS matrix offset(input offset). Range: -127 to 128
   * @param[in]      num_ch          Number of channels in LHS/RHS
   * @param[in]      out_shift       Per channel output shift. Length of vector is equal to number of channels.
   * @param[in]      out_mult        Per channel output multiplier. Length of vector is equal to number of channels.
   * @param[in]      out_offset      Offset to be added to the output values. Range: -127 to 128
   * @param[in]      activation_min  Minimum value to clamp the output to. Range: int8
   * @param[in]      activation_max  Maximum value to clamp the output to. Range: int8
   * @param[in]       row_x_col       (row_dimension * col_dimension) of LHS/RHS matrix
   * @param[in]      output_bias     Per channel output bias. Length of vector is equal to number of channels.
   * @param[in]      out             Output pointer
   *
   * @return         The function returns one of the two
   *                  - Updated output pointer if an implementaiton is available
   *                  - NULL if no implementation is available.
   *
   * @note           If number of channels is not a multiple of 4, upto 3 elements outside the boundary will be read out
   *                 for the following.
   *                  - Output shift
   *                  - Output multiplier
   *                  - Output bias
   *                  - rhs
   */
  q7_t *arm_nn_depthwise_conv_nt_t_s8(const q7_t *lhs,
                                      const q7_t *rhs,
                                      const int32_t lhs_offset,
                                      const uint16_t num_ch,
                                      const int32_t *out_shift,
                                      const int32_t *out_mult,
                                      const int32_t out_offset,
                                      const int32_t activation_min,
                                      const int32_t activation_max,
                                      const uint16_t row_x_col,
                                      const int32_t *const output_bias,
                                      q7_t *out);
  
  /**
    @brief         Read 2 q15 elements and post increment pointer.
    @param[in]     in_q15   Pointer to pointer that holds address of input.
    @return        q31 value
   */
  __STATIC_FORCEINLINE q31_t arm_nn_read_q15x2_ia(const q15_t **in_q15)
  {
    q31_t val;
  
    memcpy(&val, *in_q15, 4);
    *in_q15 += 2;
  
    return (val);
  }
  
  /**
    @brief         Read 4 q7 from q7 pointer and post increment pointer.
    @param[in]     in_q7       Pointer to pointer that holds address of input.
    @return        q31 value
   */
  __STATIC_FORCEINLINE q31_t arm_nn_read_q7x4_ia(const q7_t **in_q7)
  {
    q31_t val;
    memcpy(&val, *in_q7, 4);
    *in_q7 += 4;
  
    return (val);
  }
  
  /**
    @brief         Read 2 q15 from q15 pointer.
    @param[in]     in_q15   pointer to address of input.
    @return        q31 value
   */
  __STATIC_FORCEINLINE q31_t arm_nn_read_q15x2(const q15_t *in_q15)
  {
    q31_t val;
    memcpy(&val, in_q15, 4);
  
    return (val);
  }
  
  /**
    @brief         Read 4 q7 values.
    @param[in]     in_q7       pointer to address of input.
    @return        q31 value
   */
  __STATIC_FORCEINLINE q31_t arm_nn_read_q7x4(const q7_t *in_q7)
  {
    q31_t val;
    memcpy(&val, in_q7, 4);
  
    return (val);
  }
  
  /**
   * @brief           memset optimized for MVE
   * @param[in, out]  dst         Destination pointer
   * @param[in]       val         Value to set
   * @param[in]       block_size  Number of bytes to copy.
   *
   */
  __STATIC_FORCEINLINE void arm_memset_q7(q7_t *dst,
                                          const q7_t val,
                                          uint32_t block_size)
  {
  #if defined(ARM_MATH_MVEI)
       __asm volatile (
          "   vdup.8                  q0, %[set_val]             \n"
          "   wlstp.8                 lr, %[cnt], 1f             \n"
          "2:                                                    \n"
          "   vstrb.8                 q0, [%[in]], 16            \n"
          "   letp                    lr, 2b                     \n"
          "1:                                                    \n"
          :[in] "+r"(dst)
          :[cnt] "r"(block_size), [set_val] "r"(val)
          :"q0", "memory", "r14");
  #else
      memset(dst, val, block_size);
  #endif
  }
  
  #if defined (ARM_MATH_DSP)
  
  /**
   * @brief read and expand one q7 word into two q15 words
   */
  
  __STATIC_FORCEINLINE const q7_t *read_and_pad(const q7_t *source, q31_t * out1, q31_t * out2)
  {
          q31_t     inA = arm_nn_read_q7x4_ia(&source);
          q31_t     inAbuf1 = __SXTB16(__ROR((uint32_t)inA, 8));
          q31_t     inAbuf2 = __SXTB16(inA);
  
  #ifndef ARM_MATH_BIG_ENDIAN
    *out2 = (int32_t) (__PKHTB (inAbuf1, inAbuf2, 16));
    *out1 = (int32_t) (__PKHBT (inAbuf2, inAbuf1, 16));
  #else
    *out1 = (int32_t) (__PKHTB(inAbuf1, inAbuf2, 16));
    *out2 = (int32_t) (__PKHBT(inAbuf2, inAbuf1, 16));
  #endif
  
          return source;
  }
  
  /**
   * @brief read and expand one q7 word into two q15 words with reordering
   */
  
  __STATIC_FORCEINLINE const q7_t *read_and_pad_reordered(const q7_t *source, q31_t * out1, q31_t * out2)
  {
          q31_t     inA = arm_nn_read_q7x4_ia(&source);
  #ifndef ARM_MATH_BIG_ENDIAN
          *out2 = __SXTB16(__ROR((uint32_t)inA, 8));
          *out1 = __SXTB16(inA);
  #else
          *out1 = __SXTB16(__ROR((uint32_t)inA, 8));
          *out2 = __SXTB16(inA);
  #endif
  
          return source;
  }
  
  /**
   * @brief read and expand one q7 word into two q15 words with reordering and add an offset
   */
  __STATIC_FORCEINLINE const q7_t *read_and_pad_reordered_with_offset(const q7_t *source, q31_t * out1, q31_t * out2, q31_t offset)
  {
          q31_t     inA = arm_nn_read_q7x4_ia(&source);
  
  #ifndef ARM_MATH_BIG_ENDIAN
          *out2 = __SXTB16(__ROR((uint32_t)inA, 8));
          *out1 = __SXTB16(inA);
  #else
          *out1 = __SXTB16(__ROR((uint32_t)inA, 8));
          *out2 = __SXTB16(inA);
  #endif
          *out1 = __QADD16(*out1,offset);
          *out2 = __QADD16(*out2,offset);
  
          return source;
  }
  
  #endif
  
  
  
  /**
   * @defgroup NNBasicMath Basic Math Functions for Neural Network Computation
   *
   * Basic Math Functions for Neural Network Computation
   *
   */
  
  /**
   * @brief           q7 vector multiplication with variable output shifts
   * @param[in]       *pSrcA        pointer to the first input vector
   * @param[in]       *pSrcB        pointer to the second input vector
   * @param[out]      *pDst         pointer to the output vector
   * @param[in]       out_shift     amount of right-shift for output
   * @param[in]       blockSize     number of samples in each vector
   * @return none.
   *
   * <b>Scaling and Overflow Behavior:</b>
   * \par
   * The function uses saturating arithmetic.
   * Results outside of the allowable q15 range [0x8000 0x7FFF] will be saturated.
   */
  
  void arm_nn_mult_q15(
    q15_t * pSrcA,
    q15_t * pSrcB,
    q15_t * pDst,
    const uint16_t out_shift,
    uint32_t blockSize);
  
  /**
   * @brief           q7 vector multiplication with variable output shifts
   * @param[in]       *pSrcA        pointer to the first input vector
   * @param[in]       *pSrcB        pointer to the second input vector
   * @param[out]      *pDst         pointer to the output vector
   * @param[in]       out_shift     amount of right-shift for output
   * @param[in]       blockSize     number of samples in each vector
   * @return none.
   *
   * <b>Scaling and Overflow Behavior:</b>
   * \par
   * The function uses saturating arithmetic.
   * Results outside of the allowable q7 range [0x80 0x7F] will be saturated.
   */
  
  void arm_nn_mult_q7(
    q7_t * pSrcA,
    q7_t * pSrcB,
    q7_t * pDst,
    const uint16_t out_shift,
    uint32_t blockSize);
  
  /**
   * @brief macro for adding rounding offset
   */
  #ifndef ARM_NN_TRUNCATE
      #define NN_ROUND(out_shift) ( (0x1u << out_shift) >> 1 )
  #else
      #define NN_ROUND(out_shift) 0
  #endif
  
  // Macros for shortening quantization functions' names and avoid long lines
  #define MUL_SAT(a, b)  arm_nn_doubling_high_mult((a), (b))
  #define MUL_SAT_MVE(a, b) arm_doubling_high_mult_mve_32x4((a), (b))
  #define MUL_POW2(a, b) arm_nn_mult_by_power_of_two((a), (b))
  
  
  #define DIV_POW2(a, b) arm_nn_divide_by_power_of_two((a), (b))
  #define DIV_POW2_MVE(a, b) arm_divide_by_power_of_two_mve((a), (b))
  
  
  #define EXP_ON_NEG(x)  arm_nn_exp_on_negative_values((x))
  #define ONE_OVER1(x)   arm_nn_one_over_one_plus_x_for_x_in_0_1((x))
  
  /**
   * @brief           Saturating doubling high multiply. Result matches
   *                  NEON instruction VQRDMULH.
   * @param[in]       m1        Multiplicand. Range: {Q31_MIN, Q31_MAX}
   * @param[in]       m2        Multiplier. Range: {Q31_MIN, Q31_MAX}
   * @return          Result of multiplication.
   *
   */
  __STATIC_FORCEINLINE q31_t arm_nn_doubling_high_mult(const q31_t m1, const q31_t m2)
  {
      q31_t result = 0;
      // Rounding offset to add for a right shift of 31
      q63_t mult = 1 << 30;
  
      if ((m1 < 0) ^ (m2 < 0))
      {
          mult = 1 - mult;
      }
      // Gets resolved as a SMLAL instruction
      mult = mult + (q63_t)m1 * m2;
  
      // Utilize all of the upper 32 bits. This is the doubling step
      // as well.
      result = (int32_t) (mult / (1ll << 31));
  
      if ((m1 == m2) && (m1 == (int32_t)Q31_MIN))
      {
          result = Q31_MAX;
      }
      return result;
  }
  
  /**
   * @brief           Doubling high multiply without saturation. This is intended
   *                  for requantization where the scale is a positive integer
   *
   * @param[in]       m1        Multiplicand. Range: {Q31_MIN, Q31_MAX}
   * @param[in]       m2        Multiplier Range: {Q31_MIN, Q31_MAX}
   * @return          Result of multiplication.
   * @note            The result of this matches that of neon instruction
   *                  VQRDMULH for m1 in range {Q31_MIN, Q31_MAX} and m2 in
   *                  range {Q31_MIN + 1, Q31_MAX}. Saturation occurs when
   *                  m1 equals m2 equals Q31_MIN and that is not handled by
   *                  this function.
   *
   */
  __STATIC_FORCEINLINE q31_t arm_nn_doubling_high_mult_no_sat(const q31_t m1, const q31_t m2)
  {
      q31_t result = 0;
      union arm_nn_long_long mult;
  
      // Rounding offset to add for a right shift of 31
      mult.word.low = 1 << 30;
      mult.word.high = 0;
  
      // Gets resolved as a SMLAL instruction
      mult.long_long = mult.long_long + (q63_t)m1 * m2;
  
      // Utilize all of the upper 32 bits. This is the doubling step
      // as well.
      result = (int32_t)(mult.long_long >> 31);
  
      return result;
  }
  
  /**
   * @brief           Rounding divide by power of two.
   * @param[in]       dividend - Dividend
   * @param[in]       exponent - Divisor = power(2, exponent)
   *                             Range: [0, 31]
   * @return          Rounded result of division. Midpoint is rounded away from zero.
   *
   */
  __STATIC_FORCEINLINE q31_t arm_nn_divide_by_power_of_two(const q31_t dividend, const q31_t exponent)
  {
      q31_t result = 0;
      const q31_t remainder_mask = (1 << exponent) - 1;
      int32_t remainder = remainder_mask & dividend;
  
      // Basic division
      result = dividend >> exponent;
  
      // Adjust 'result' for rounding (mid point away from zero)
      q31_t threshold = remainder_mask >> 1;
      if (result < 0)
      {
          threshold++;
      }
      if (remainder > threshold)
      {
          result++;
      }
  
      return result;
  }
  
  /**
   * @brief           Requantize a given value.
   * @param[in]       val         Value to be requantized
   * @param[in]       multiplier  multiplier. Range {Q31_MIN + 1, Q32_MAX}
   * @param[in]       shift       left or right shift for 'val * multiplier'
   *
   * @return          Returns (val * multiplier)/(2 ^ shift)
   *
   */
  __STATIC_FORCEINLINE q31_t arm_nn_requantize(const q31_t val, const q31_t multiplier, const q31_t shift)
  {
    return arm_nn_divide_by_power_of_two(
        arm_nn_doubling_high_mult_no_sat(val * (1 << LEFT_SHIFT(shift)), multiplier),
        RIGHT_SHIFT(shift));
  }
  
  /**
   * @brief           memcpy optimized for MVE
   * @param[in, out]  dst         Destination pointer
   * @param[in]       src         Source pointer.
   * @param[in]       block_size  Number of bytes to copy.
   *
   */
  __STATIC_FORCEINLINE void arm_memcpy_q7(q7_t *__RESTRICT dst,
                                          const q7_t *__RESTRICT src,
                                          uint32_t block_size)
  {
  #if defined(ARM_MATH_MVEI)
       __asm volatile (
          "   wlstp.8                 lr, %[cnt], 1f             \n"
          "2:                                                    \n"
          "   vldrb.8                 q0, [%[in]], 16            \n"
          "   vstrb.8                 q0, [%[out]], 16           \n"
          "   letp                    lr, 2b                     \n"
          "1:                                                    \n"
          :[in] "+r"(src)
          ,[out] "+r"(dst)
          :[cnt] "r"(block_size)
          :"q0", "memory", "r14");
  #else
      memcpy(dst, src, block_size);
  #endif
  }
  
  #if defined(ARM_MATH_MVEI)
  /**
   * @brief           Vector saturating doubling high multiply returning high half.
   * @param[in]       m1        Multiplicand
   * @param[in]       m2        Multiplier
   * @return          Result of multiplication.
   *
   */
  __STATIC_FORCEINLINE int32x4_t arm_doubling_high_mult_mve(const int32x4_t m1, const q31_t m2)
  {
      return vqrdmulhq_n_s32(m1, m2);
  }
  
  /**
   * @brief           Vector rounding divide by power of two.
   * @param[in]       dividend - Dividend vector
   * @param[in]       exponent - Divisor = power(2, exponent)
   *                             Range: [0, 31]
   * @return          Rounded result of division. Midpoint is rounded away from zero.
   *
   */
  __STATIC_FORCEINLINE int32x4_t arm_divide_by_power_of_two_mve(const int32x4_t dividend, const q31_t exponent)
  {
    const int32x4_t shift = vdupq_n_s32(-exponent);
    const int32x4_t fixup = vshrq_n_s32(vandq_s32(dividend, shift), 31);
    const int32x4_t fixed_up_dividend = vqaddq_s32(dividend, fixup);
    return vrshlq_s32(fixed_up_dividend, shift);
  }
  
  /**
   * @brief           Requantize a given vector.
   * @param[in]       val         Vector to be requantized
   * @param[in]       multiplier  multiplier
   * @param[in]       shift       shift
   *
   * @return          Returns (val * multiplier)/(2 ^ shift)
   *
   */
  __STATIC_FORCEINLINE int32x4_t arm_requantize_mve(const int32x4_t val, const q31_t multiplier, const q31_t shift)
  {
    return arm_divide_by_power_of_two_mve(
            arm_doubling_high_mult_mve(vshlq_s32(val, vdupq_n_s32(LEFT_SHIFT(shift))), multiplier),
            RIGHT_SHIFT(shift));
  }
  
  __STATIC_FORCEINLINE int32x4_t arm_doubling_high_mult_mve_32x4(const int32x4_t m1, const int32x4_t m2)
  {
    return vqrdmulhq_s32(m1, m2);
  }
  
  __STATIC_FORCEINLINE int32x4_t arm_divide_by_power_of_two_mve_32x4(const int32x4_t dividend, const int32x4_t exponent)
  {
    const int32x4_t shift = -exponent;
    const int32x4_t fixup = vshrq_n_s32(vandq_s32(dividend, shift), 31);
    const int32x4_t fixed_up_dividend = vqaddq_s32(dividend, fixup);
    return vrshlq_s32(fixed_up_dividend, shift);
  }
  
  __STATIC_FORCEINLINE int32x4_t arm_requantize_mve_32x4(const int32x4_t val, const int32x4_t multiplier, const int32x4_t shift)
  {
    const int32x4_t zz = vdupq_n_s32(0);
    const mve_pred16_t p = vcmpgtq_n_s32(shift, 0);
  
    const int32x4_t left_shift = vpselq_s32(shift, zz, p);
    const int32x4_t right_shift = -vpselq_s32(zz, shift, p);
  
    return arm_divide_by_power_of_two_mve_32x4(arm_doubling_high_mult_mve_32x4(vshlq_s32(val, left_shift), multiplier), right_shift);
  }
  #endif
  
  // @note The following functions are used only for softmax layer, scaled bits = 5 assumed
  
  __STATIC_FORCEINLINE int32_t arm_nn_exp_on_negative_values(int32_t val)
  {
      int32_t mask  = 0;
      int32_t shift = 24;
  
      const int32_t val_mod_minus_quarter = (val & ((1 << shift) - 1)) - (1 << shift);
      const int32_t remainder             = val_mod_minus_quarter - val;
      const int32_t x                     = (val_mod_minus_quarter << 5) + (1 << 28);
      const int32_t x2                    = MUL_SAT(x, x);
  
      int32_t result = 1895147668 + MUL_SAT(1895147668, x +
          DIV_POW2(MUL_SAT(DIV_POW2(MUL_SAT(x2, x2), 2) + MUL_SAT(x2, x), 715827883) + x2, 1));
  
  #define SELECT_IF_NON_ZERO(x)                                     \
  {                                                                 \
      mask   = MASK_IF_NON_ZERO(remainder & (1 << shift++));        \
      result = SELECT_USING_MASK(mask, MUL_SAT(result, x), result); \
  }
  
      SELECT_IF_NON_ZERO(1672461947)
      SELECT_IF_NON_ZERO(1302514674)
      SELECT_IF_NON_ZERO(790015084)
      SELECT_IF_NON_ZERO(290630308)
      SELECT_IF_NON_ZERO(39332535)
      SELECT_IF_NON_ZERO(720401)
      SELECT_IF_NON_ZERO(242)
  
  #undef SELECT_IF_NON_ZERO
  
      mask = MASK_IF_ZERO(val);
      return SELECT_USING_MASK(mask, Q31_MAX, result);
  }
  
  __STATIC_FORCEINLINE q31_t arm_nn_mult_by_power_of_two(const int32_t val, const int32_t exp)
  {
      const int32_t thresh = ((1 << (31 - exp)) - 1);
      int32_t result = val << exp;
      result = SELECT_USING_MASK(MASK_IF_NON_ZERO(val > thresh), Q31_MAX, result);
      result = SELECT_USING_MASK(MASK_IF_NON_ZERO(val < -thresh), Q31_MIN, result);
      return result;
  }
  
  __STATIC_FORCEINLINE int32_t arm_nn_one_over_one_plus_x_for_x_in_0_1(int32_t val)
  {
      const int64_t sum = (int64_t)val + (int64_t)Q31_MAX;
      const int32_t half_denominator = (int32_t)((sum + (sum >= 0 ? 1 : -1)) / 2L);
      int32_t x = 1515870810 + MUL_SAT(half_denominator, -1010580540);
  
      const int32_t shift = (1 << 29);
      x += MUL_POW2(MUL_SAT(x, shift - MUL_SAT(half_denominator, x)), 2);
      x += MUL_POW2(MUL_SAT(x, shift - MUL_SAT(half_denominator, x)), 2);
      x += MUL_POW2(MUL_SAT(x, shift - MUL_SAT(half_denominator, x)), 2);
  
      return MUL_POW2(x, 1);
  }
  
  #ifdef __cplusplus
  }
  #endif
  
  #endif
  