text: /*
  * Copyright(c) 2019 Intel Corporation
  *
  * This source code is subject to the terms of the BSD 2 Clause License and
  * the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
  * was not distributed with this source code in the LICENSE file, you can
  * obtain it at https://www.aomedia.org/license/software-license. If the Alliance for Open
  * Media Patent License 1.0 was not distributed with this source code in the
  * PATENTS file, you can obtain it at https://www.aomedia.org/license/patent-license.
  */
  
  #include "EbDefinitions.h"
  
  #if EN_AVX512_SUPPORT
  
  #include <immintrin.h>
  #include "common_dsp_rtcd.h"
  #include "convolve.h"
  #include "convolve_avx2.h"
  #include "convolve_avx512.h"
  #include "EbInterPrediction.h"
  #include "EbMemory_AVX2.h"
  #include "synonyms.h"
  
  static INLINE __m512i sr_x_round_avx512(const __m512i src) {
      const __m512i round = _mm512_set1_epi16(34);
      const __m512i dst   = _mm512_add_epi16(src, round);
      return _mm512_srai_epi16(dst, 6);
  }
  
  static INLINE void sr_x_round_store_32x2_avx512(const __m512i res[2], uint8_t *const dst,
                                                  const int32_t dst_stride) {
      __m512i r[2];
  
      r[0] = sr_x_round_avx512(res[0]);
      r[1] = sr_x_round_avx512(res[1]);
      convolve_store_32x2_avx512(r[0], r[1], dst, dst_stride);
  }
  
  static INLINE void sr_x_round_store_64_avx512(const __m512i res[2], uint8_t *const dst) {
      __m512i r[2];
  
      r[0] = sr_x_round_avx512(res[0]);
      r[1] = sr_x_round_avx512(res[1]);
      convolve_store_64_avx512(r[0], r[1], dst);
  }
  
  static INLINE void sr_y_round_store_64_avx512(const __m512i res[2], uint8_t *const dst) {
      __m512i r[2];
  
      r[0] = sr_y_round_avx512(res[0]);
      r[1] = sr_y_round_avx512(res[1]);
      convolve_store_64_avx512(r[0], r[1], dst);
  }
  
  static INLINE void sr_y_round_store_64x2_avx512(const __m512i res[4], uint8_t *const dst,
                                                  const int32_t dst_stride) {
      sr_y_round_store_64_avx512(res, dst);
      sr_y_round_store_64_avx512(res + 2, dst + dst_stride);
  }
  
  static INLINE void sr_y_round_store_32x2_avx512(const __m512i res[2], uint8_t *const dst,
                                                  const int32_t dst_stride) {
      __m512i r[2];
  
      r[0] = sr_y_round_avx512(res[0]);
      r[1] = sr_y_round_avx512(res[1]);
      convolve_store_32x2_avx512(r[0], r[1], dst, dst_stride);
  }
  
  static INLINE void sr_y_2tap_32x2_avx512(const uint8_t *const src, const int32_t src_stride,
                                           const __m512i coeffs[1], __m256i *const s,
                                           uint8_t *const dst, const int32_t dst_stride) {
      __m512i r[2];
      y_convolve_2tap_32x2_avx512(src, src_stride, coeffs, s, r);
      sr_y_round_store_32x2_avx512(r, dst, dst_stride);
  }
  
  static INLINE void sr_y_2tap_64_avx512(const uint8_t *const src, const __m512i coeffs[1],
                                         const __m512i s0, __m512i *const s1, uint8_t *const dst) {
      __m512i r[2];
      y_convolve_2tap_64_avx512(src, coeffs, s0, s1, r);
      sr_y_round_store_64_avx512(r, dst);
  }
  
  static INLINE void sr_y_2tap_64_avg_avx512(const uint8_t *const src, const __m512i s0,
                                             __m512i *const s1, uint8_t *const dst) {
      *s1             = _mm512_loadu_si512((__m512i *)src);
      const __m512i d = _mm512_avg_epu8(s0, *s1);
      _mm512_storeu_si512((__m512i *)dst, d);
  }
  
  void svt_av1_convolve_y_sr_avx512(const uint8_t *src, int32_t src_stride, uint8_t *dst,
                                    int32_t dst_stride, int32_t w, int32_t h,
                                    InterpFilterParams *filter_params_x,
                                    InterpFilterParams *filter_params_y, const int32_t subpel_x_q4,
                                    const int32_t subpel_y_q4, ConvolveParams *conv_params) {
      int32_t x, y;
      __m128i coeffs_128[4];
      __m256i coeffs_256[4];
      __m512i coeffs_512[4];
  
      (void)filter_params_x;
      (void)subpel_x_q4;
      (void)conv_params;
  
      if (is_convolve_2tap(filter_params_y->filter_ptr)) {
          // vert_filt as 2 tap
          const uint8_t *src_ptr = src;
  
          y = h;
  
          if (subpel_y_q4 != 8) {
              if (w <= 8) {
                  prepare_half_coeffs_2tap_ssse3(filter_params_y, subpel_y_q4, coeffs_128);
  
                  if (w == 2) {
                      __m128i s_16[2];
  
                      s_16[0] = _mm_cvtsi32_si128(*(int16_t *)src_ptr);
  
                      do {
                          const __m128i res = y_convolve_2tap_2x2_ssse3(
                              src_ptr, src_stride, coeffs_128, s_16);
                          const __m128i r = sr_y_round_sse2(res);
                          pack_store_2x2_sse2(r, dst, dst_stride);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  } else if (w == 4) {
                      __m128i s_32[2];
  
                      s_32[0] = _mm_cvtsi32_si128(*(int32_t *)src_ptr);
  
                      do {
                          const __m128i res = y_convolve_2tap_4x2_ssse3(
                              src_ptr, src_stride, coeffs_128, s_32);
                          const __m128i r = sr_y_round_sse2(res);
                          pack_store_4x2_sse2(r, dst, dst_stride);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  } else {
                      __m128i s_64[2], s_128[2];
  
                      assert(w == 8);
  
                      s_64[0] = _mm_loadl_epi64((__m128i *)src_ptr);
  
                      do {
                          // Note: Faster than binding to AVX2 registers.
                          s_64[1]            = _mm_loadl_epi64((__m128i *)(src_ptr + src_stride));
                          s_128[0]           = _mm_unpacklo_epi64(s_64[0], s_64[1]);
                          s_64[0]            = _mm_loadl_epi64((__m128i *)(src_ptr + 2 * src_stride));
                          s_128[1]           = _mm_unpacklo_epi64(s_64[1], s_64[0]);
                          const __m128i ss0  = _mm_unpacklo_epi8(s_128[0], s_128[1]);
                          const __m128i ss1  = _mm_unpackhi_epi8(s_128[0], s_128[1]);
                          const __m128i res0 = convolve_2tap_ssse3(&ss0, coeffs_128);
                          const __m128i res1 = convolve_2tap_ssse3(&ss1, coeffs_128);
                          const __m128i r0   = sr_y_round_sse2(res0);
                          const __m128i r1   = sr_y_round_sse2(res1);
                          const __m128i d    = _mm_packus_epi16(r0, r1);
                          _mm_storel_epi64((__m128i *)dst, d);
                          _mm_storeh_epi64((__m128i *)(dst + dst_stride), d);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  }
              } else if (w == 16) {
                  __m128i s_128[2];
  
                  prepare_half_coeffs_2tap_avx2(filter_params_y, subpel_y_q4, coeffs_256);
  
                  s_128[0] = _mm_loadu_si128((__m128i *)src_ptr);
  
                  do {
                      __m256i r[2];
  
                      y_convolve_2tap_16x2_avx2(src_ptr, src_stride, coeffs_256, s_128, r);
                      sr_y_round_store_16x2_avx2(r, dst, dst_stride);
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else {
                  prepare_half_coeffs_2tap_avx512(filter_params_y, subpel_y_q4, coeffs_512);
  
                  if (w == 32) {
                      __m256i s_256 = _mm256_loadu_si256((__m256i *)src_ptr);
  
                      do {
                          sr_y_2tap_32x2_avx512(
                              src_ptr + src_stride, src_stride, coeffs_512, &s_256, dst, dst_stride);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  } else if (w == 64) {
                      __m512i s_512[2];
  
                      s_512[0] = _mm512_loadu_si512((__m512i *)src_ptr);
  
                      do {
                          sr_y_2tap_64_avx512(
                              src_ptr + src_stride, coeffs_512, s_512[0], &s_512[1], dst);
                          sr_y_2tap_64_avx512(src_ptr + 2 * src_stride,
                                              coeffs_512,
                                              s_512[1],
                                              &s_512[0],
                                              dst + dst_stride);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  } else {
                      __m512i s_512[2][2];
  
                      assert(w == 128);
  
                      s_512[0][0] = _mm512_loadu_si512((__m512i *)(src_ptr + 0 * 64));
                      s_512[0][1] = _mm512_loadu_si512((__m512i *)(src_ptr + 1 * 64));
  
                      do {
                          sr_y_2tap_64_avx512(src_ptr + 1 * src_stride + 0 * 64,
                                              coeffs_512,
                                              s_512[0][0],
                                              &s_512[1][0],
                                              dst);
                          sr_y_2tap_64_avx512(src_ptr + 1 * src_stride + 1 * 64,
                                              coeffs_512,
                                              s_512[0][1],
                                              &s_512[1][1],
                                              dst + 2 * 32);
  
                          sr_y_2tap_64_avx512(src_ptr + 2 * src_stride + 0 * 64,
                                              coeffs_512,
                                              s_512[1][0],
                                              &s_512[0][0],
                                              dst + dst_stride);
                          sr_y_2tap_64_avx512(src_ptr + 2 * src_stride + 1 * 64,
                                              coeffs_512,
                                              s_512[1][1],
                                              &s_512[0][1],
                                              dst + dst_stride + 2 * 32);
  
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  }
              }
          } else {
              // average to get half pel
              if (w <= 8) {
                  if (w == 2) {
                      __m128i s_16[2];
  
                      s_16[0] = _mm_cvtsi32_si128(*(int16_t *)src_ptr);
  
                      do {
                          s_16[1]          = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + src_stride));
                          const __m128i d0 = _mm_avg_epu8(s_16[0], s_16[1]);
                          *(int16_t *)dst  = (int16_t)_mm_cvtsi128_si32(d0);
                          s_16[0] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 2 * src_stride));
                          const __m128i d1               = _mm_avg_epu8(s_16[1], s_16[0]);
                          *(int16_t *)(dst + dst_stride) = (int16_t)_mm_cvtsi128_si32(d1);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  } else if (w == 4) {
                      __m128i s_32[2];
  
                      s_32[0] = _mm_cvtsi32_si128(*(int32_t *)src_ptr);
  
                      do {
                          s_32[1]          = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + src_stride));
                          const __m128i d0 = _mm_avg_epu8(s_32[0], s_32[1]);
                          xx_storel_32(dst, d0);
                          s_32[0] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 2 * src_stride));
                          const __m128i d1 = _mm_avg_epu8(s_32[1], s_32[0]);
                          xx_storel_32(dst + dst_stride, d1);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  } else {
                      __m128i s_64[2];
  
                      assert(w == 8);
  
                      s_64[0] = _mm_loadl_epi64((__m128i *)src_ptr);
  
                      do {
                          // Note: Faster than binding to AVX2 registers.
                          s_64[1]          = _mm_loadl_epi64((__m128i *)(src_ptr + src_stride));
                          const __m128i d0 = _mm_avg_epu8(s_64[0], s_64[1]);
                          _mm_storel_epi64((__m128i *)dst, d0);
                          s_64[0]          = _mm_loadl_epi64((__m128i *)(src_ptr + 2 * src_stride));
                          const __m128i d1 = _mm_avg_epu8(s_64[1], s_64[0]);
                          _mm_storel_epi64((__m128i *)(dst + dst_stride), d1);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  }
              } else if (w == 16) {
                  __m128i s_128[2];
  
                  s_128[0] = _mm_loadu_si128((__m128i *)src_ptr);
  
                  do {
                      s_128[1]         = _mm_loadu_si128((__m128i *)(src_ptr + src_stride));
                      const __m128i d0 = _mm_avg_epu8(s_128[0], s_128[1]);
                      _mm_storeu_si128((__m128i *)dst, d0);
                      s_128[0]         = _mm_loadu_si128((__m128i *)(src_ptr + 2 * src_stride));
                      const __m128i d1 = _mm_avg_epu8(s_128[1], s_128[0]);
                      _mm_storeu_si128((__m128i *)(dst + dst_stride), d1);
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else if (w == 32) {
                  __m256i s_256[2];
  
                  s_256[0] = _mm256_loadu_si256((__m256i *)src_ptr);
  
                  do {
                      sr_y_2tap_32_avg_avx2(src_ptr + src_stride, s_256[0], &s_256[1], dst);
                      sr_y_2tap_32_avg_avx2(
                          src_ptr + 2 * src_stride, s_256[1], &s_256[0], dst + dst_stride);
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else if (w == 64) {
                  __m512i s_512[2];
  
                  s_512[0] = _mm512_loadu_si512((__m512i *)src_ptr);
  
                  do {
                      sr_y_2tap_64_avg_avx512(src_ptr + src_stride, s_512[0], &s_512[1], dst);
                      sr_y_2tap_64_avg_avx512(
                          src_ptr + 2 * src_stride, s_512[1], &s_512[0], dst + dst_stride);
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else {
                  __m512i s_512[2][2];
  
                  assert(w == 128);
  
                  s_512[0][0] = _mm512_loadu_si512((__m512i *)(src_ptr + 0 * 64));
                  s_512[0][1] = _mm512_loadu_si512((__m512i *)(src_ptr + 1 * 64));
  
                  do {
                      sr_y_2tap_64_avg_avx512(
                          src_ptr + src_stride + 0 * 6, s_512[0][0], &s_512[1][0], dst + 0 * 6);
                      sr_y_2tap_64_avg_avx512(
                          src_ptr + src_stride + 1 * 64, s_512[0][1], &s_512[1][1], dst + 1 * 64);
  
                      sr_y_2tap_64_avg_avx512(src_ptr + 2 * src_stride + 0 * 6,
                                              s_512[1][0],
                                              &s_512[0][0],
                                              dst + dst_stride + 0 * 6);
                      sr_y_2tap_64_avg_avx512(src_ptr + 2 * src_stride + 1 * 64,
                                              s_512[1][1],
                                              &s_512[0][1],
                                              dst + dst_stride + 1 * 64);
  
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              }
          }
      } else if (is_convolve_4tap(filter_params_y->filter_ptr)) {
          // vert_filt as 4 tap
          const uint8_t *src_ptr = src - src_stride;
  
          y = h;
  
          if (w <= 4) {
              prepare_half_coeffs_4tap_ssse3(filter_params_y, subpel_y_q4, coeffs_128);
  
              if (w == 2) {
                  __m128i s_16[4], ss_128[2];
  
                  s_16[0] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 0 * src_stride));
                  s_16[1] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 1 * src_stride));
                  s_16[2] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 2 * src_stride));
  
                  const __m128i src01 = _mm_unpacklo_epi16(s_16[0], s_16[1]);
                  const __m128i src12 = _mm_unpacklo_epi16(s_16[1], s_16[2]);
  
                  ss_128[0] = _mm_unpacklo_epi8(src01, src12);
  
                  do {
                      src_ptr += 2 * src_stride;
                      const __m128i res = y_convolve_4tap_2x2_ssse3(
                          src_ptr, src_stride, coeffs_128, s_16, ss_128);
                      const __m128i r = sr_y_round_sse2(res);
                      pack_store_2x2_sse2(r, dst, dst_stride);
  
                      ss_128[0] = ss_128[1];
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else {
                  __m128i s_32[4], ss_128[2];
  
                  assert(w == 4);
  
                  s_32[0] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 0 * src_stride));
                  s_32[1] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 1 * src_stride));
                  s_32[2] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 2 * src_stride));
  
                  const __m128i src01 = _mm_unpacklo_epi32(s_32[0], s_32[1]);
                  const __m128i src12 = _mm_unpacklo_epi32(s_32[1], s_32[2]);
  
                  ss_128[0] = _mm_unpacklo_epi8(src01, src12);
  
                  do {
                      src_ptr += 2 * src_stride;
                      const __m128i res = y_convolve_4tap_4x2_ssse3(
                          src_ptr, src_stride, coeffs_128, s_32, ss_128);
                      const __m128i r = sr_y_round_sse2(res);
                      pack_store_4x2_sse2(r, dst, dst_stride);
  
                      ss_128[0] = ss_128[1];
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              }
          } else {
              prepare_half_coeffs_4tap_avx2(filter_params_y, subpel_y_q4, coeffs_256);
  
              if (w == 8) {
                  __m128i s_64[4];
                  __m256i ss_256[2];
  
                  s_64[0] = _mm_loadl_epi64((__m128i *)(src_ptr + 0 * src_stride));
                  s_64[1] = _mm_loadl_epi64((__m128i *)(src_ptr + 1 * src_stride));
                  s_64[2] = _mm_loadl_epi64((__m128i *)(src_ptr + 2 * src_stride));
  
                  // Load lines a and b. Line a to lower 128, line b to upper 128
                  const __m256i src01 = _mm256_setr_m128i(s_64[0], s_64[1]);
                  const __m256i src12 = _mm256_setr_m128i(s_64[1], s_64[2]);
  
                  ss_256[0] = _mm256_unpacklo_epi8(src01, src12);
  
                  do {
                      src_ptr += 2 * src_stride;
                      const __m256i res = y_convolve_4tap_8x2_avx2(
                          src_ptr, src_stride, coeffs_256, s_64, ss_256);
                      sr_y_round_store_8x2_avx2(res, dst, dst_stride);
  
                      ss_256[0] = ss_256[1];
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else if (w == 16) {
                  __m128i s_128[4];
                  __m256i ss_256[4], r[2];
  
                  assert(w == 16);
  
                  s_128[0] = _mm_loadu_si128((__m128i *)(src_ptr + 0 * src_stride));
                  s_128[1] = _mm_loadu_si128((__m128i *)(src_ptr + 1 * src_stride));
                  s_128[2] = _mm_loadu_si128((__m128i *)(src_ptr + 2 * src_stride));
  
                  // Load lines a and b. Line a to lower 128, line b to upper 128
                  const __m256i src01 = _mm256_setr_m128i(s_128[0], s_128[1]);
                  const __m256i src12 = _mm256_setr_m128i(s_128[1], s_128[2]);
  
                  ss_256[0] = _mm256_unpacklo_epi8(src01, src12);
                  ss_256[2] = _mm256_unpackhi_epi8(src01, src12);
  
                  do {
                      src_ptr += 2 * src_stride;
                      y_convolve_4tap_16x2_avx2(src_ptr, src_stride, coeffs_256, s_128, ss_256, r);
                      sr_y_round_store_16x2_avx2(r, dst, dst_stride);
  
                      ss_256[0] = ss_256[1];
                      ss_256[2] = ss_256[3];
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else {
                  // AV1 standard won't have 32x4 case.
                  // This only favors some optimization feature which
                  // subsamples 32x8 to 32x4 and triggers 4-tap filter.
  
                  __m256i s_256[3];
                  __m512i s_512[2], ss_512[4], r[2];
  
                  assert(w == 32);
  
                  prepare_half_coeffs_4tap_avx512(filter_params_y, subpel_y_q4, coeffs_512);
  
                  s_256[0] = _mm256_loadu_si256((__m256i *)(src_ptr + 0 * src_stride));
                  s_256[1] = _mm256_loadu_si256((__m256i *)(src_ptr + 1 * src_stride));
                  s_256[2] = _mm256_loadu_si256((__m256i *)(src_ptr + 2 * src_stride));
  
                  s_512[0] = _mm512_setr_m256i(s_256[0], s_256[1]);
                  s_512[1] = _mm512_setr_m256i(s_256[1], s_256[2]);
  
                  ss_512[0] = _mm512_unpacklo_epi8(s_512[0], s_512[1]);
                  ss_512[2] = _mm512_unpackhi_epi8(s_512[0], s_512[1]);
  
                  do {
                      src_ptr += 2 * src_stride;
                      y_convolve_4tap_32x2_avx512(src_ptr, src_stride, coeffs_512, s_256, ss_512, r);
                      sr_y_round_store_32x2_avx512(r, dst, dst_stride);
  
                      ss_512[0] = ss_512[1];
                      ss_512[2] = ss_512[3];
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              }
          }
      } else if (is_convolve_6tap(filter_params_y->filter_ptr)) {
          // vert_filt as 6 tap
          const uint8_t *src_ptr = src - 2 * src_stride;
  
          if (w <= 4) {
              prepare_half_coeffs_6tap_ssse3(filter_params_y, subpel_y_q4, coeffs_128);
  
              y = h;
  
              if (w == 2) {
                  __m128i s_16[6], ss_128[3];
  
                  s_16[0] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 0 * src_stride));
                  s_16[1] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 1 * src_stride));
                  s_16[2] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 2 * src_stride));
                  s_16[3] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 3 * src_stride));
                  s_16[4] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 4 * src_stride));
  
                  const __m128i src01 = _mm_unpacklo_epi16(s_16[0], s_16[1]);
                  const __m128i src12 = _mm_unpacklo_epi16(s_16[1], s_16[2]);
                  const __m128i src23 = _mm_unpacklo_epi16(s_16[2], s_16[3]);
                  const __m128i src34 = _mm_unpacklo_epi16(s_16[3], s_16[4]);
  
                  ss_128[0] = _mm_unpacklo_epi8(src01, src12);
                  ss_128[1] = _mm_unpacklo_epi8(src23, src34);
  
                  do {
                      src_ptr += 2 * src_stride;
                      const __m128i res = y_convolve_6tap_2x2_ssse3(
                          src_ptr, src_stride, coeffs_128, s_16, ss_128);
                      const __m128i r = sr_y_round_sse2(res);
                      pack_store_2x2_sse2(r, dst, dst_stride);
  
                      ss_128[0] = ss_128[1];
                      ss_128[1] = ss_128[2];
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else {
                  __m128i s_32[6], ss_128[3];
  
                  assert(w == 4);
  
                  s_32[0] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 0 * src_stride));
                  s_32[1] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 1 * src_stride));
                  s_32[2] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 2 * src_stride));
                  s_32[3] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 3 * src_stride));
                  s_32[4] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 4 * src_stride));
  
                  const __m128i src01 = _mm_unpacklo_epi32(s_32[0], s_32[1]);
                  const __m128i src12 = _mm_unpacklo_epi32(s_32[1], s_32[2]);
                  const __m128i src23 = _mm_unpacklo_epi32(s_32[2], s_32[3]);
                  const __m128i src34 = _mm_unpacklo_epi32(s_32[3], s_32[4]);
  
                  ss_128[0] = _mm_unpacklo_epi8(src01, src12);
                  ss_128[1] = _mm_unpacklo_epi8(src23, src34);
  
                  do {
                      src_ptr += 2 * src_stride;
                      const __m128i res = y_convolve_6tap_4x2_ssse3(
                          src_ptr, src_stride, coeffs_128, s_32, ss_128);
                      const __m128i r = sr_y_round_sse2(res);
                      pack_store_4x2_sse2(r, dst, dst_stride);
  
                      ss_128[0] = ss_128[1];
                      ss_128[1] = ss_128[2];
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              }
          } else if (w <= 16) {
              prepare_half_coeffs_6tap_avx2(filter_params_y, subpel_y_q4, coeffs_256);
  
              if (w == 8) {
                  __m128i s_64[6];
                  __m256i ss_256[3];
  
                  s_64[0] = _mm_loadl_epi64((__m128i *)(src_ptr + 0 * src_stride));
                  s_64[1] = _mm_loadl_epi64((__m128i *)(src_ptr + 1 * src_stride));
                  s_64[2] = _mm_loadl_epi64((__m128i *)(src_ptr + 2 * src_stride));
                  s_64[3] = _mm_loadl_epi64((__m128i *)(src_ptr + 3 * src_stride));
                  s_64[4] = _mm_loadl_epi64((__m128i *)(src_ptr + 4 * src_stride));
  
                  // Load lines a and b. Line a to lower 128, line b to upper 128
                  const __m256i src01 = _mm256_setr_m128i(s_64[0], s_64[1]);
                  const __m256i src12 = _mm256_setr_m128i(s_64[1], s_64[2]);
                  const __m256i src23 = _mm256_setr_m128i(s_64[2], s_64[3]);
                  const __m256i src34 = _mm256_setr_m128i(s_64[3], s_64[4]);
  
                  ss_256[0] = _mm256_unpacklo_epi8(src01, src12);
                  ss_256[1] = _mm256_unpacklo_epi8(src23, src34);
  
                  y = h;
                  do {
                      src_ptr += 2 * src_stride;
                      const __m256i res = y_convolve_6tap_8x2_avx2(
                          src_ptr, src_stride, coeffs_256, s_64, ss_256);
                      sr_y_round_store_8x2_avx2(res, dst, dst_stride);
  
                      ss_256[0] = ss_256[1];
                      ss_256[1] = ss_256[2];
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else {
                  __m128i s_128[6];
                  __m256i ss_256[6], r[2];
  
                  assert(w == 16);
  
                  s_128[0] = _mm_loadu_si128((__m128i *)(src_ptr + 0 * src_stride));
                  s_128[1] = _mm_loadu_si128((__m128i *)(src_ptr + 1 * src_stride));
                  s_128[2] = _mm_loadu_si128((__m128i *)(src_ptr + 2 * src_stride));
                  s_128[3] = _mm_loadu_si128((__m128i *)(src_ptr + 3 * src_stride));
                  s_128[4] = _mm_loadu_si128((__m128i *)(src_ptr + 4 * src_stride));
  
                  // Load lines a and b. Line a to lower 128, line b to upper 128
                  const __m256i src01 = _mm256_setr_m128i(s_128[0], s_128[1]);
                  const __m256i src12 = _mm256_setr_m128i(s_128[1], s_128[2]);
                  const __m256i src23 = _mm256_setr_m128i(s_128[2], s_128[3]);
                  const __m256i src34 = _mm256_setr_m128i(s_128[3], s_128[4]);
  
                  ss_256[0] = _mm256_unpacklo_epi8(src01, src12);
                  ss_256[1] = _mm256_unpacklo_epi8(src23, src34);
  
                  ss_256[3] = _mm256_unpackhi_epi8(src01, src12);
                  ss_256[4] = _mm256_unpackhi_epi8(src23, src34);
  
                  y = h;
                  do {
                      src_ptr += 2 * src_stride;
                      y_convolve_6tap_16x2_avx2(src_ptr, src_stride, coeffs_256, s_128, ss_256, r);
                      sr_y_round_store_16x2_avx2(r, dst, dst_stride);
  
                      ss_256[0] = ss_256[1];
                      ss_256[1] = ss_256[2];
  
                      ss_256[3] = ss_256[4];
                      ss_256[4] = ss_256[5];
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              }
          } else {
              prepare_half_coeffs_6tap_avx512(filter_params_y, subpel_y_q4, coeffs_512);
  
              if (w == 32) {
                  __m256i s_256[5];
                  __m512i s_512[4], ss_512[6], r[2];
  
                  s_256[0] = _mm256_loadu_si256((__m256i *)(src_ptr + 0 * src_stride));
                  s_256[1] = _mm256_loadu_si256((__m256i *)(src_ptr + 1 * src_stride));
                  s_256[2] = _mm256_loadu_si256((__m256i *)(src_ptr + 2 * src_stride));
                  s_256[3] = _mm256_loadu_si256((__m256i *)(src_ptr + 3 * src_stride));
                  s_256[4] = _mm256_loadu_si256((__m256i *)(src_ptr + 4 * src_stride));
  
                  s_512[0] = _mm512_setr_m256i(s_256[0], s_256[1]);
                  s_512[1] = _mm512_setr_m256i(s_256[1], s_256[2]);
                  s_512[2] = _mm512_setr_m256i(s_256[2], s_256[3]);
                  s_512[3] = _mm512_setr_m256i(s_256[3], s_256[4]);
  
                  ss_512[0] = _mm512_unpacklo_epi8(s_512[0], s_512[1]);
                  ss_512[1] = _mm512_unpacklo_epi8(s_512[2], s_512[3]);
                  ss_512[3] = _mm512_unpackhi_epi8(s_512[0], s_512[1]);
                  ss_512[4] = _mm512_unpackhi_epi8(s_512[2], s_512[3]);
  
                  y = h;
                  do {
                      src_ptr += 2 * src_stride;
                      y_convolve_6tap_32x2_avx512(src_ptr, src_stride, coeffs_512, s_256, ss_512, r);
                      sr_y_round_store_32x2_avx512(r, dst, dst_stride);
  
                      ss_512[0] = ss_512[1];
                      ss_512[1] = ss_512[2];
                      ss_512[3] = ss_512[4];
                      ss_512[4] = ss_512[5];
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else {
                  __m512i s_512[6], ss_512[6], tt_512[6], r[4];
  
                  assert(!(w % 64));
  
                  x = 0;
                  do {
                      const uint8_t *s = src_ptr + x;
                      uint8_t       *d = dst + x;
  
                      s_512[0] = _mm512_loadu_si512((__m512i *)(s + 0 * src_stride));
                      s_512[1] = _mm512_loadu_si512((__m512i *)(s + 1 * src_stride));
                      s_512[2] = _mm512_loadu_si512((__m512i *)(s + 2 * src_stride));
                      s_512[3] = _mm512_loadu_si512((__m512i *)(s + 3 * src_stride));
                      s_512[4] = _mm512_loadu_si512((__m512i *)(s + 4 * src_stride));
  
                      ss_512[0] = _mm512_unpacklo_epi8(s_512[0], s_512[1]);
                      ss_512[1] = _mm512_unpacklo_epi8(s_512[2], s_512[3]);
                      ss_512[3] = _mm512_unpackhi_epi8(s_512[0], s_512[1]);
                      ss_512[4] = _mm512_unpackhi_epi8(s_512[2], s_512[3]);
  
                      tt_512[0] = _mm512_unpacklo_epi8(s_512[1], s_512[2]);
                      tt_512[1] = _mm512_unpacklo_epi8(s_512[3], s_512[4]);
                      tt_512[3] = _mm512_unpackhi_epi8(s_512[1], s_512[2]);
                      tt_512[4] = _mm512_unpackhi_epi8(s_512[3], s_512[4]);
  
                      y = h;
                      do {
                          s += 2 * src_stride;
                          y_convolve_6tap_64x2_avx512(
                              s, src_stride, coeffs_512, s_512, ss_512, tt_512, r);
                          sr_y_round_store_64x2_avx512(r, d, dst_stride);
  
                          ss_512[0] = ss_512[1];
                          ss_512[1] = ss_512[2];
                          ss_512[3] = ss_512[4];
                          ss_512[4] = ss_512[5];
  
                          tt_512[0] = tt_512[1];
                          tt_512[1] = tt_512[2];
                          tt_512[3] = tt_512[4];
                          tt_512[4] = tt_512[5];
                          d += 2 * dst_stride;
                          y -= 2;
                      } while (y);
  
                      x += 64;
                  } while (x < w);
              }
          }
      } else {
          // vert_filt as 8 tap
          const uint8_t *src_ptr = src - 3 * src_stride;
  
          if (w <= 4) {
              prepare_half_coeffs_8tap_ssse3(filter_params_y, subpel_y_q4, coeffs_128);
  
              y = h;
  
              if (w == 2) {
                  __m128i s_16[8], ss_128[4];
  
                  s_16[0] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 0 * src_stride));
                  s_16[1] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 1 * src_stride));
                  s_16[2] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 2 * src_stride));
                  s_16[3] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 3 * src_stride));
                  s_16[4] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 4 * src_stride));
                  s_16[5] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 5 * src_stride));
                  s_16[6] = _mm_cvtsi32_si128(*(int16_t *)(src_ptr + 6 * src_stride));
  
                  const __m128i src01 = _mm_unpacklo_epi16(s_16[0], s_16[1]);
                  const __m128i src12 = _mm_unpacklo_epi16(s_16[1], s_16[2]);
                  const __m128i src23 = _mm_unpacklo_epi16(s_16[2], s_16[3]);
                  const __m128i src34 = _mm_unpacklo_epi16(s_16[3], s_16[4]);
                  const __m128i src45 = _mm_unpacklo_epi16(s_16[4], s_16[5]);
                  const __m128i src56 = _mm_unpacklo_epi16(s_16[5], s_16[6]);
  
                  ss_128[0] = _mm_unpacklo_epi8(src01, src12);
                  ss_128[1] = _mm_unpacklo_epi8(src23, src34);
                  ss_128[2] = _mm_unpacklo_epi8(src45, src56);
  
                  do {
                      const __m128i res = y_convolve_8tap_2x2_ssse3(
                          src_ptr, src_stride, coeffs_128, s_16, ss_128);
                      const __m128i r = sr_y_round_sse2(res);
                      pack_store_2x2_sse2(r, dst, dst_stride);
                      ss_128[0] = ss_128[1];
                      ss_128[1] = ss_128[2];
                      ss_128[2] = ss_128[3];
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else {
                  __m128i s_32[8], ss_128[4];
  
                  assert(w == 4);
  
                  s_32[0] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 0 * src_stride));
                  s_32[1] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 1 * src_stride));
                  s_32[2] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 2 * src_stride));
                  s_32[3] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 3 * src_stride));
                  s_32[4] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 4 * src_stride));
                  s_32[5] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 5 * src_stride));
                  s_32[6] = _mm_cvtsi32_si128(*(int32_t *)(src_ptr + 6 * src_stride));
  
                  const __m128i src01 = _mm_unpacklo_epi32(s_32[0], s_32[1]);
                  const __m128i src12 = _mm_unpacklo_epi32(s_32[1], s_32[2]);
                  const __m128i src23 = _mm_unpacklo_epi32(s_32[2], s_32[3]);
                  const __m128i src34 = _mm_unpacklo_epi32(s_32[3], s_32[4]);
                  const __m128i src45 = _mm_unpacklo_epi32(s_32[4], s_32[5]);
                  const __m128i src56 = _mm_unpacklo_epi32(s_32[5], s_32[6]);
  
                  ss_128[0] = _mm_unpacklo_epi8(src01, src12);
                  ss_128[1] = _mm_unpacklo_epi8(src23, src34);
                  ss_128[2] = _mm_unpacklo_epi8(src45, src56);
  
                  do {
                      const __m128i res = y_convolve_8tap_4x2_ssse3(
                          src_ptr, src_stride, coeffs_128, s_32, ss_128);
                      const __m128i r = sr_y_round_sse2(res);
                      pack_store_4x2_sse2(r, dst, dst_stride);
                      ss_128[0] = ss_128[1];
                      ss_128[1] = ss_128[2];
                      ss_128[2] = ss_128[3];
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              }
          } else if (w <= 16) {
              prepare_half_coeffs_8tap_avx2(filter_params_y, subpel_y_q4, coeffs_256);
  
              if (w == 8) {
                  __m128i s_64[8];
                  __m256i ss_256[4];
  
                  s_64[0] = _mm_loadl_epi64((__m128i *)(src_ptr + 0 * src_stride));
                  s_64[1] = _mm_loadl_epi64((__m128i *)(src_ptr + 1 * src_stride));
                  s_64[2] = _mm_loadl_epi64((__m128i *)(src_ptr + 2 * src_stride));
                  s_64[3] = _mm_loadl_epi64((__m128i *)(src_ptr + 3 * src_stride));
                  s_64[4] = _mm_loadl_epi64((__m128i *)(src_ptr + 4 * src_stride));
                  s_64[5] = _mm_loadl_epi64((__m128i *)(src_ptr + 5 * src_stride));
                  s_64[6] = _mm_loadl_epi64((__m128i *)(src_ptr + 6 * src_stride));
  
                  // Load lines a and b. Line a to lower 128, line b to upper 128
                  const __m256i src01 = _mm256_setr_m128i(s_64[0], s_64[1]);
                  const __m256i src12 = _mm256_setr_m128i(s_64[1], s_64[2]);
                  const __m256i src23 = _mm256_setr_m128i(s_64[2], s_64[3]);
                  const __m256i src34 = _mm256_setr_m128i(s_64[3], s_64[4]);
                  const __m256i src45 = _mm256_setr_m128i(s_64[4], s_64[5]);
                  const __m256i src56 = _mm256_setr_m128i(s_64[5], s_64[6]);
  
                  ss_256[0] = _mm256_unpacklo_epi8(src01, src12);
                  ss_256[1] = _mm256_unpacklo_epi8(src23, src34);
                  ss_256[2] = _mm256_unpacklo_epi8(src45, src56);
  
                  y = h;
                  do {
                      const __m256i res = y_convolve_8tap_8x2_avx2(
                          src_ptr, src_stride, coeffs_256, s_64, ss_256);
                      sr_y_round_store_8x2_avx2(res, dst, dst_stride);
                      ss_256[0] = ss_256[1];
                      ss_256[1] = ss_256[2];
                      ss_256[2] = ss_256[3];
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else {
                  __m128i s_128[8];
                  __m256i ss_256[8], r[2];
  
                  assert(w == 16);
  
                  s_128[0] = _mm_loadu_si128((__m128i *)(src_ptr + 0 * src_stride));
                  s_128[1] = _mm_loadu_si128((__m128i *)(src_ptr + 1 * src_stride));
                  s_128[2] = _mm_loadu_si128((__m128i *)(src_ptr + 2 * src_stride));
                  s_128[3] = _mm_loadu_si128((__m128i *)(src_ptr + 3 * src_stride));
                  s_128[4] = _mm_loadu_si128((__m128i *)(src_ptr + 4 * src_stride));
                  s_128[5] = _mm_loadu_si128((__m128i *)(src_ptr + 5 * src_stride));
                  s_128[6] = _mm_loadu_si128((__m128i *)(src_ptr + 6 * src_stride));
  
                  // Load lines a and b. Line a to lower 128, line b to upper 128
                  const __m256i src01 = _mm256_setr_m128i(s_128[0], s_128[1]);
                  const __m256i src12 = _mm256_setr_m128i(s_128[1], s_128[2]);
                  const __m256i src23 = _mm256_setr_m128i(s_128[2], s_128[3]);
                  const __m256i src34 = _mm256_setr_m128i(s_128[3], s_128[4]);
                  const __m256i src45 = _mm256_setr_m128i(s_128[4], s_128[5]);
                  const __m256i src56 = _mm256_setr_m128i(s_128[5], s_128[6]);
  
                  ss_256[0] = _mm256_unpacklo_epi8(src01, src12);
                  ss_256[1] = _mm256_unpacklo_epi8(src23, src34);
                  ss_256[2] = _mm256_unpacklo_epi8(src45, src56);
  
                  ss_256[4] = _mm256_unpackhi_epi8(src01, src12);
                  ss_256[5] = _mm256_unpackhi_epi8(src23, src34);
                  ss_256[6] = _mm256_unpackhi_epi8(src45, src56);
  
                  y = h;
                  do {
                      y_convolve_8tap_16x2_avx2(src_ptr, src_stride, coeffs_256, s_128, ss_256, r);
                      sr_y_round_store_16x2_avx2(r, dst, dst_stride);
  
                      ss_256[0] = ss_256[1];
                      ss_256[1] = ss_256[2];
                      ss_256[2] = ss_256[3];
  
                      ss_256[4] = ss_256[5];
                      ss_256[5] = ss_256[6];
                      ss_256[6] = ss_256[7];
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              }
          } else {
              prepare_half_coeffs_8tap_avx512(filter_params_y, subpel_y_q4, coeffs_512);
  
              if (w == 32) {
                  __m256i s_256[7];
                  __m512i s_512[6], ss_512[8], r[2];
  
                  s_256[0] = _mm256_loadu_si256((__m256i *)(src_ptr + 0 * src_stride));
                  s_256[1] = _mm256_loadu_si256((__m256i *)(src_ptr + 1 * src_stride));
                  s_256[2] = _mm256_loadu_si256((__m256i *)(src_ptr + 2 * src_stride));
                  s_256[3] = _mm256_loadu_si256((__m256i *)(src_ptr + 3 * src_stride));
                  s_256[4] = _mm256_loadu_si256((__m256i *)(src_ptr + 4 * src_stride));
                  s_256[5] = _mm256_loadu_si256((__m256i *)(src_ptr + 5 * src_stride));
                  s_256[6] = _mm256_loadu_si256((__m256i *)(src_ptr + 6 * src_stride));
  
                  s_512[0] = _mm512_setr_m256i(s_256[0], s_256[1]);
                  s_512[1] = _mm512_setr_m256i(s_256[1], s_256[2]);
                  s_512[2] = _mm512_setr_m256i(s_256[2], s_256[3]);
                  s_512[3] = _mm512_setr_m256i(s_256[3], s_256[4]);
                  s_512[4] = _mm512_setr_m256i(s_256[4], s_256[5]);
                  s_512[5] = _mm512_setr_m256i(s_256[5], s_256[6]);
  
                  ss_512[0] = _mm512_unpacklo_epi8(s_512[0], s_512[1]);
                  ss_512[1] = _mm512_unpacklo_epi8(s_512[2], s_512[3]);
                  ss_512[2] = _mm512_unpacklo_epi8(s_512[4], s_512[5]);
                  ss_512[4] = _mm512_unpackhi_epi8(s_512[0], s_512[1]);
                  ss_512[5] = _mm512_unpackhi_epi8(s_512[2], s_512[3]);
                  ss_512[6] = _mm512_unpackhi_epi8(s_512[4], s_512[5]);
  
                  y = h;
                  do {
                      src_ptr += 2 * src_stride;
                      y_convolve_8tap_32x2_avx512(src_ptr, src_stride, coeffs_512, s_256, ss_512, r);
                      sr_y_round_store_32x2_avx512(r, dst, dst_stride);
  
                      ss_512[0] = ss_512[1];
                      ss_512[1] = ss_512[2];
                      ss_512[2] = ss_512[3];
                      ss_512[4] = ss_512[5];
                      ss_512[5] = ss_512[6];
                      ss_512[6] = ss_512[7];
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else {
                  __m512i s_512[8], ss_512[8], tt_512[8], r[4];
  
                  assert(!(w % 64));
  
                  x = 0;
                  do {
                      const uint8_t *s = src_ptr + x;
                      uint8_t       *d = dst + x;
  
                      s_512[0] = _mm512_loadu_si512((__m512i *)(s + 0 * src_stride));
                      s_512[1] = _mm512_loadu_si512((__m512i *)(s + 1 * src_stride));
                      s_512[2] = _mm512_loadu_si512((__m512i *)(s + 2 * src_stride));
                      s_512[3] = _mm512_loadu_si512((__m512i *)(s + 3 * src_stride));
                      s_512[4] = _mm512_loadu_si512((__m512i *)(s + 4 * src_stride));
                      s_512[5] = _mm512_loadu_si512((__m512i *)(s + 5 * src_stride));
                      s_512[6] = _mm512_loadu_si512((__m512i *)(s + 6 * src_stride));
  
                      ss_512[0] = _mm512_unpacklo_epi8(s_512[0], s_512[1]);
                      ss_512[1] = _mm512_unpacklo_epi8(s_512[2], s_512[3]);
                      ss_512[2] = _mm512_unpacklo_epi8(s_512[4], s_512[5]);
                      ss_512[4] = _mm512_unpackhi_epi8(s_512[0], s_512[1]);
                      ss_512[5] = _mm512_unpackhi_epi8(s_512[2], s_512[3]);
                      ss_512[6] = _mm512_unpackhi_epi8(s_512[4], s_512[5]);
  
                      tt_512[0] = _mm512_unpacklo_epi8(s_512[1], s_512[2]);
                      tt_512[1] = _mm512_unpacklo_epi8(s_512[3], s_512[4]);
                      tt_512[2] = _mm512_unpacklo_epi8(s_512[5], s_512[6]);
                      tt_512[4] = _mm512_unpackhi_epi8(s_512[1], s_512[2]);
                      tt_512[5] = _mm512_unpackhi_epi8(s_512[3], s_512[4]);
                      tt_512[6] = _mm512_unpackhi_epi8(s_512[5], s_512[6]);
  
                      y = h;
                      do {
                          y_convolve_8tap_64x2_avx512(
                              s, src_stride, coeffs_512, s_512, ss_512, tt_512, r);
                          sr_y_round_store_64x2_avx512(r, d, dst_stride);
  
                          ss_512[0] = ss_512[1];
                          ss_512[1] = ss_512[2];
                          ss_512[2] = ss_512[3];
                          ss_512[4] = ss_512[5];
                          ss_512[5] = ss_512[6];
                          ss_512[6] = ss_512[7];
  
                          tt_512[0] = tt_512[1];
                          tt_512[1] = tt_512[2];
                          tt_512[2] = tt_512[3];
                          tt_512[4] = tt_512[5];
                          tt_512[5] = tt_512[6];
                          tt_512[6] = tt_512[7];
                          s += 2 * src_stride;
                          d += 2 * dst_stride;
                          y -= 2;
                      } while (y);
  
                      x += 64;
                  } while (x < w);
              }
          }
      }
  }
  
  static INLINE void sr_x_2tap_32x2_avx512(const uint8_t *const src, const int32_t src_stride,
                                           const __m512i coeffs[1], uint8_t *const dst,
                                           const int32_t dst_stride) {
      __m512i r[2];
  
      x_convolve_2tap_32x2_avx512(src, src_stride, coeffs, r);
      sr_x_round_store_32x2_avx512(r, dst, dst_stride);
  }
  
  static INLINE void sr_x_2tap_64_avx512(const uint8_t *const src, const __m512i coeffs[1],
                                         uint8_t *const dst) {
      __m512i r[2];
  
      x_convolve_2tap_64_avx512(src, coeffs, r);
      sr_x_round_store_64_avx512(r, dst);
  }
  
  static INLINE void sr_x_2tap_64_avg_avx512(const uint8_t *const src, uint8_t *const dst) {
      const __m512i s0 = _mm512_loadu_si512((__m512i *)src);
      const __m512i s1 = _mm512_loadu_si512((__m512i *)(src + 1));
      const __m512i d  = _mm512_avg_epu8(s0, s1);
      _mm512_storeu_si512((__m512i *)dst, d);
  }
  
  static INLINE void sr_x_6tap_32x2_avx512(const uint8_t *const src, const int32_t src_stride,
                                           const __m512i coeffs[3], const __m512i filt[3],
                                           uint8_t *const dst, const int32_t dst_stride) {
      __m512i r[2];
  
      x_convolve_6tap_32x2_avx512(src, src_stride, coeffs, filt, r);
      sr_x_round_store_32x2_avx512(r, dst, dst_stride);
  }
  
  static INLINE void sr_x_6tap_64_avx512(const uint8_t *const src, const __m512i coeffs[3],
                                         const __m512i filt[3], uint8_t *const dst) {
      __m512i r[2];
  
      x_convolve_6tap_64_avx512(src, coeffs, filt, r);
      sr_x_round_store_64_avx512(r, dst);
  }
  
  SIMD_INLINE void sr_x_8tap_32x2_avx512(const uint8_t *const src, const int32_t src_stride,
                                         const __m512i coeffs[4], const __m512i filt[4],
                                         uint8_t *const dst, const int32_t dst_stride) {
      __m512i r[2];
  
      x_convolve_8tap_32x2_avx512(src, src_stride, coeffs, filt, r);
      sr_x_round_store_32x2_avx512(r, dst, dst_stride);
  }
  
  SIMD_INLINE void sr_x_8tap_64_avx512(const uint8_t *const src, const __m512i coeffs[4],
                                       const __m512i filt[4], uint8_t *const dst) {
      __m512i r[2];
  
      x_convolve_8tap_64_avx512(src, coeffs, filt, r);
      sr_x_round_store_64_avx512(r, dst);
  }
  
  void svt_av1_convolve_x_sr_avx512(const uint8_t *src, int32_t src_stride, uint8_t *dst,
                                    int32_t dst_stride, int32_t w, int32_t h,
                                    InterpFilterParams *filter_params_x,
                                    InterpFilterParams *filter_params_y, const int32_t subpel_x_q4,
                                    const int32_t subpel_y_q4, ConvolveParams *conv_params) {
      int32_t y = h;
      __m128i coeffs_128[4];
      __m256i coeffs_256[4];
      __m512i coeffs_512[4];
  
      (void)filter_params_y;
      (void)subpel_y_q4;
      (void)conv_params;
  
      assert(conv_params->round_0 == 3);
      assert((FILTER_BITS - conv_params->round_1) >= 0 ||
             ((conv_params->round_0 + conv_params->round_1) == 2 * FILTER_BITS));
  
      if (is_convolve_2tap(filter_params_x->filter_ptr)) {
          // horz_filt as 2 tap
          const uint8_t *src_ptr = src;
  
          if (subpel_x_q4 != 8) {
              if (w <= 8) {
                  prepare_half_coeffs_2tap_ssse3(filter_params_x, subpel_x_q4, coeffs_128);
  
                  if (w == 2) {
                      do {
                          const __m128i res = x_convolve_2tap_2x2_sse4_1(
                              src_ptr, src_stride, coeffs_128);
                          const __m128i r = sr_x_round_sse2(res);
                          pack_store_2x2_sse2(r, dst, dst_stride);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  } else if (w == 4) {
                      do {
                          const __m128i res = x_convolve_2tap_4x2_ssse3(
                              src_ptr, src_stride, coeffs_128);
                          const __m128i r = sr_x_round_sse2(res);
                          pack_store_4x2_sse2(r, dst, dst_stride);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  } else {
                      assert(w == 8);
  
                      do {
                          __m128i res[2];
  
                          x_convolve_2tap_8x2_ssse3(src_ptr, src_stride, coeffs_128, res);
                          res[0]          = sr_x_round_sse2(res[0]);
                          res[1]          = sr_x_round_sse2(res[1]);
                          const __m128i d = _mm_packus_epi16(res[0], res[1]);
                          _mm_storel_epi64((__m128i *)dst, d);
                          _mm_storeh_epi64((__m128i *)(dst + dst_stride), d);
  
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  }
              } else if (w == 16) {
                  prepare_half_coeffs_2tap_avx2(filter_params_x, subpel_x_q4, coeffs_256);
  
                  do {
                      __m256i r[2];
  
                      x_convolve_2tap_16x2_avx2(src_ptr, src_stride, coeffs_256, r);
                      sr_x_round_store_16x2_avx2(r, dst, dst_stride);
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else {
                  prepare_half_coeffs_2tap_avx512(filter_params_x, subpel_x_q4, coeffs_512);
  
                  if (w == 32) {
                      do {
                          // Slower than avx2.
                          sr_x_2tap_32x2_avx512(src_ptr, src_stride, coeffs_512, dst, dst_stride);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  } else if (w == 64) {
                      do {
                          sr_x_2tap_64_avx512(src_ptr, coeffs_512, dst);
                          src_ptr += src_stride;
                          dst += dst_stride;
                      } while (--y);
                  } else {
                      assert(w == 128);
  
                      do {
                          sr_x_2tap_64_avx512(src_ptr + 0 * 64, coeffs_512, dst + 0 * 64);
                          sr_x_2tap_64_avx512(src_ptr + 1 * 64, coeffs_512, dst + 1 * 64);
                          src_ptr += src_stride;
                          dst += dst_stride;
                      } while (--y);
                  }
              }
          } else {
              // average to get half pel
              if (w == 2) {
                  do {
                      __m128i s_128;
  
                      s_128                           = load_u8_4x2_sse4_1(src_ptr, src_stride);
                      const __m128i s1                = _mm_srli_si128(s_128, 1);
                      const __m128i d                 = _mm_avg_epu8(s_128, s1);
                      *(uint16_t *)dst                = (uint16_t)_mm_cvtsi128_si32(d);
                      *(uint16_t *)(dst + dst_stride) = _mm_extract_epi16(d, 2);
  
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else if (w == 4) {
                  do {
                      __m128i s_128;
  
                      s_128            = load_u8_8x2_sse2(src_ptr, src_stride);
                      const __m128i s1 = _mm_srli_si128(s_128, 1);
                      const __m128i d  = _mm_avg_epu8(s_128, s1);
                      xx_storel_32(dst, d);
                      *(int32_t *)(dst + dst_stride) = _mm_extract_epi32(d, 2);
  
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else if (w == 8) {
                  do {
                      const __m128i s00 = _mm_loadu_si128((__m128i *)src_ptr);
                      const __m128i s10 = _mm_loadu_si128((__m128i *)(src_ptr + src_stride));
                      const __m128i s01 = _mm_srli_si128(s00, 1);
                      const __m128i s11 = _mm_srli_si128(s10, 1);
                      const __m128i d0  = _mm_avg_epu8(s00, s01);
                      const __m128i d1  = _mm_avg_epu8(s10, s11);
                      _mm_storel_epi64((__m128i *)dst, d0);
                      _mm_storel_epi64((__m128i *)(dst + dst_stride), d1);
  
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else if (w == 16) {
                  do {
                      const __m128i s00 = _mm_loadu_si128((__m128i *)src_ptr);
                      const __m128i s01 = _mm_loadu_si128((__m128i *)(src_ptr + 1));
                      const __m128i s10 = _mm_loadu_si128((__m128i *)(src_ptr + src_stride));
                      const __m128i s11 = _mm_loadu_si128((__m128i *)(src_ptr + src_stride + 1));
                      const __m128i d0  = _mm_avg_epu8(s00, s01);
                      const __m128i d1  = _mm_avg_epu8(s10, s11);
                      _mm_storeu_si128((__m128i *)dst, d0);
                      _mm_storeu_si128((__m128i *)(dst + dst_stride), d1);
  
                      src_ptr += 2 * src_stride;
                      dst += 2 * dst_stride;
                      y -= 2;
                  } while (y);
              } else if (w == 32) {
                  do {
                      sr_x_2tap_32_avg_avx2(src_ptr, dst);
                      src_ptr += src_stride;
                      dst += dst_stride;
                  } while (--y);
              } else if (w == 64) {
                  do {
                      sr_x_2tap_64_avg_avx512(src_ptr, dst);
                      src_ptr += src_stride;
                      dst += dst_stride;
                  } while (--y);
              } else {
                  assert(w == 128);
  
                  do {
                      sr_x_2tap_64_avg_avx512(src_ptr + 0 * 64, dst + 0 * 64);
                      sr_x_2tap_64_avg_avx512(src_ptr + 1 * 64, dst + 1 * 64);
                      src_ptr += src_stride;
                      dst += dst_stride;
                  } while (--y);
              }
          }
      } else if (is_convolve_4tap(filter_params_x->filter_ptr)) {
          // horz_filt as 4 tap
          const uint8_t *src_ptr = src - 1;
  
          prepare_half_coeffs_4tap_ssse3(filter_params_x, subpel_x_q4, coeffs_128);
  
          if (w == 2) {
              do {
                  const __m128i res = x_convolve_4tap_2x2_ssse3(src_ptr, src_stride, coeffs_128);
                  const __m128i r   = sr_x_round_sse2(res);
                  pack_store_2x2_sse2(r, dst, dst_stride);
                  src_ptr += 2 * src_stride;
                  dst += 2 * dst_stride;
                  y -= 2;
              } while (y);
          } else {
              assert(w == 4);
  
              do {
                  const __m128i res = x_convolve_4tap_4x2_ssse3(src_ptr, src_stride, coeffs_128);
                  const __m128i r   = sr_x_round_sse2(res);
                  pack_store_4x2_sse2(r, dst, dst_stride);
                  src_ptr += 2 * src_stride;
                  dst += 2 * dst_stride;
                  y -= 2;
              } while (y);
          }
      } else {
          if (is_convolve_6tap(filter_params_x->filter_ptr)) {
              // horz_filt as 6 tap
              const uint8_t *src_ptr = src - 2;
  
              if (w <= 16) {
                  __m256i filt_256[3];
  
                  prepare_half_coeffs_6tap_avx2(filter_params_x, subpel_x_q4, coeffs_256);
  
                  filt_256[0] = _mm256_loadu_si256((__m256i const *)filt1_global_avx);
                  filt_256[1] = _mm256_loadu_si256((__m256i const *)filt2_global_avx);
                  filt_256[2] = _mm256_loadu_si256((__m256i const *)filt3_global_avx);
  
                  if (w == 8) {
                      do {
                          const __m256i res = x_convolve_6tap_8x2_avx2(
                              src_ptr, src_stride, coeffs_256, filt_256);
                          sr_x_round_store_8x2_avx2(res, dst, dst_stride);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  } else {
                      assert(w == 16);
  
                      do {
                          __m256i r[2];
  
                          x_convolve_6tap_16x2_avx2(src_ptr, src_stride, coeffs_256, filt_256, r);
                          sr_x_round_store_16x2_avx2(r, dst, dst_stride);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  }
              } else {
                  __m512i filt_512[3];
  
                  prepare_half_coeffs_6tap_avx512(filter_params_x, subpel_x_q4, coeffs_512);
  
                  filt_512[0] = zz_load_512(filt1_global_avx);
                  filt_512[1] = zz_load_512(filt2_global_avx);
                  filt_512[2] = zz_load_512(filt3_global_avx);
  
                  if (w == 32) {
                      do {
                          sr_x_6tap_32x2_avx512(
                              src_ptr, src_stride, coeffs_512, filt_512, dst, dst_stride);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  } else if (w == 64) {
                      do {
                          sr_x_6tap_64_avx512(src_ptr, coeffs_512, filt_512, dst);
                          src_ptr += src_stride;
                          dst += dst_stride;
                      } while (--y);
                  } else {
                      assert(w == 128);
  
                      do {
                          sr_x_6tap_64_avx512(src_ptr, coeffs_512, filt_512, dst);
                          sr_x_6tap_64_avx512(src_ptr + 64, coeffs_512, filt_512, dst + 64);
                          src_ptr += src_stride;
                          dst += dst_stride;
                      } while (--y);
                  }
              }
          } else {
              // horz_filt as 8 tap
              const uint8_t *src_ptr = src - 3;
  
              if (w <= 16) {
                  __m256i filt_256[4];
  
                  prepare_half_coeffs_8tap_avx2(filter_params_x, subpel_x_q4, coeffs_256);
  
                  filt_256[3] = _mm256_loadu_si256((__m256i const *)filt4_global_avx);
                  filt_256[0] = _mm256_loadu_si256((__m256i const *)filt1_global_avx);
                  filt_256[1] = _mm256_loadu_si256((__m256i const *)filt2_global_avx);
                  filt_256[2] = _mm256_loadu_si256((__m256i const *)filt3_global_avx);
  
                  if (w == 8) {
                      do {
                          const __m256i res = x_convolve_8tap_8x2_avx2(
                              src_ptr, src_stride, coeffs_256, filt_256);
                          sr_x_round_store_8x2_avx2(res, dst, dst_stride);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  } else {
                      assert(w == 16);
  
                      do {
                          __m256i r[2];
  
                          x_convolve_8tap_16x2_avx2(src_ptr, src_stride, coeffs_256, filt_256, r);
                          sr_x_round_store_16x2_avx2(r, dst, dst_stride);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  }
              } else {
                  __m512i filt_512[4];
  
                  prepare_half_coeffs_8tap_avx512(filter_params_x, subpel_x_q4, coeffs_512);
  
                  filt_512[0] = zz_load_512(filt1_global_avx);
                  filt_512[1] = zz_load_512(filt2_global_avx);
                  filt_512[2] = zz_load_512(filt3_global_avx);
                  filt_512[3] = zz_load_512(filt4_global_avx);
  
                  if (w == 32) {
                      do {
                          sr_x_8tap_32x2_avx512(
                              src_ptr, src_stride, coeffs_512, filt_512, dst, dst_stride);
                          src_ptr += 2 * src_stride;
                          dst += 2 * dst_stride;
                          y -= 2;
                      } while (y);
                  } else if (w == 64) {
                      do {
                          sr_x_8tap_64_avx512(src_ptr, coeffs_512, filt_512, dst);
                          src_ptr += src_stride;
                          dst += dst_stride;
                      } while (--y);
                  } else {
                      assert(w == 128);
  
                      do {
                          sr_x_8tap_64_avx512(src_ptr, coeffs_512, filt_512, dst);
                          sr_x_8tap_64_avx512(src_ptr + 64, coeffs_512, filt_512, dst + 64);
                          src_ptr += src_stride;
                          dst += dst_stride;
                      } while (--y);
                  }
              }
          }
      }
  }
  
  #endif // EN_AVX512_SUPPORT
  