text: // Copyright (c) 2018 The Bitcoin Core developers
  // Copyright (c) 2017 The Raven Core developers
  // Copyright (c) 2018 The Liquidcash Core developers
  // Distributed under the MIT software license, see the accompanying
  // file COPYING or http://www.opensource.org/licenses/mit-license.php.
  /* $Id: simd.c 227 2010-06-16 17:28:38Z tp $ */
  /*
   * SIMD implementation.
   *
   * ==========================(LICENSE BEGIN)============================
   *
   * Copyright (c) 2007-2010  Projet RNRT SAPHIR
   * 
   * Permission is hereby granted, free of charge, to any person obtaining
   * a copy of this software and associated documentation files (the
   * "Software"), to deal in the Software without restriction, including
   * without limitation the rights to use, copy, modify, merge, publish,
   * distribute, sublicense, and/or sell copies of the Software, and to
   * permit persons to whom the Software is furnished to do so, subject to
   * the following conditions:
   * 
   * The above copyright notice and this permission notice shall be
   * included in all copies or substantial portions of the Software.
   * 
   * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
   * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
   * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
   * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
   * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
   * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
   * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
   *
   * ===========================(LICENSE END)=============================
   *
   * @author   Thomas Pornin <thomas.pornin@cryptolog.com>
   */
  
  #include <stddef.h>
  #include <string.h>
  #include <limits.h>
  
  #include "sph_simd.h"
  
  #ifdef __cplusplus
  extern "C"{
  #endif
  
  #if SPH_SMALL_FOOTPRINT && !defined SPH_SMALL_FOOTPRINT_SIMD
  #define SPH_SMALL_FOOTPRINT_SIMD   1
  #endif
  
  #ifdef _MSC_VER
  #pragma warning (disable: 4146)
  #endif
  
  typedef sph_u32 u32;
  typedef sph_s32 s32;
  #define C32     SPH_C32
  #define T32     SPH_T32
  #define ROL32   SPH_ROTL32
  
  #define XCAT(x, y)    XCAT_(x, y)
  #define XCAT_(x, y)   x ## y
  
  /*
   * The powers of 41 modulo 257. We use exponents from 0 to 255, inclusive.
   */
  static const s32 alpha_tab[] = {
  	  1,  41, 139,  45,  46,  87, 226,  14,  60, 147, 116, 130,
  	190,  80, 196,  69,   2,  82,  21,  90,  92, 174, 195,  28,
  	120,  37, 232,   3, 123, 160, 135, 138,   4, 164,  42, 180,
  	184,  91, 133,  56, 240,  74, 207,   6, 246,  63,  13,  19,
  	  8,  71,  84, 103, 111, 182,   9, 112, 223, 148, 157,  12,
  	235, 126,  26,  38,  16, 142, 168, 206, 222, 107,  18, 224,
  	189,  39,  57,  24, 213, 252,  52,  76,  32,  27,  79, 155,
  	187, 214,  36, 191, 121,  78, 114,  48, 169, 247, 104, 152,
  	 64,  54, 158,  53, 117, 171,  72, 125, 242, 156, 228,  96,
  	 81, 237, 208,  47, 128, 108,  59, 106, 234,  85, 144, 250,
  	227,  55, 199, 192, 162, 217, 159,  94, 256, 216, 118, 212,
  	211, 170,  31, 243, 197, 110, 141, 127,  67, 177,  61, 188,
  	255, 175, 236, 167, 165,  83,  62, 229, 137, 220,  25, 254,
  	134,  97, 122, 119, 253,  93, 215,  77,  73, 166, 124, 201,
  	 17, 183,  50, 251,  11, 194, 244, 238, 249, 186, 173, 154,
  	146,  75, 248, 145,  34, 109, 100, 245,  22, 131, 231, 219,
  	241, 115,  89,  51,  35, 150, 239,  33,  68, 218, 200, 233,
  	 44,   5, 205, 181, 225, 230, 178, 102,  70,  43, 221,  66,
  	136, 179, 143, 209,  88,  10, 153, 105, 193, 203,  99, 204,
  	140,  86, 185, 132,  15, 101,  29, 161, 176,  20,  49, 210,
  	129, 149, 198, 151,  23, 172, 113,   7,  30, 202,  58,  65,
  	 95,  40,  98, 163
  };
  
  /*
   * Ranges:
   *   REDS1: from -32768..98302 to -383..383
   *   REDS2: from -2^31..2^31-1 to -32768..98302
   */
  #define REDS1(x)    (((x) & 0xFF) - ((x) >> 8))
  #define REDS2(x)    (((x) & 0xFFFF) + ((x) >> 16))
  
  /*
   * If, upon entry, the values of q[] are all in the -N..N range (where
   * N >= 98302) then the new values of q[] are in the -2N..2N range.
   *
   * Since alpha_tab[v] <= 256, maximum allowed range is for N = 8388608.
   */
  #define FFT_LOOP(rb, hk, as, id)   do { \
  		size_t u, v; \
  		s32 m = q[(rb)]; \
  		s32 n = q[(rb) + (hk)]; \
  		q[(rb)] = m + n; \
  		q[(rb) + (hk)] = m - n; \
  		u = v = 0; \
  		goto id; \
  		for (; u < (hk); u += 4, v += 4 * (as)) { \
  			s32 t; \
  			m = q[(rb) + u + 0]; \
  			n = q[(rb) + u + 0 + (hk)]; \
  			t = REDS2(n * alpha_tab[v + 0 * (as)]); \
  			q[(rb) + u + 0] = m + t; \
  			q[(rb) + u + 0 + (hk)] = m - t; \
  		id: \
  			m = q[(rb) + u + 1]; \
  			n = q[(rb) + u + 1 + (hk)]; \
  			t = REDS2(n * alpha_tab[v + 1 * (as)]); \
  			q[(rb) + u + 1] = m + t; \
  			q[(rb) + u + 1 + (hk)] = m - t; \
  			m = q[(rb) + u + 2]; \
  			n = q[(rb) + u + 2 + (hk)]; \
  			t = REDS2(n * alpha_tab[v + 2 * (as)]); \
  			q[(rb) + u + 2] = m + t; \
  			q[(rb) + u + 2 + (hk)] = m - t; \
  			m = q[(rb) + u + 3]; \
  			n = q[(rb) + u + 3 + (hk)]; \
  			t = REDS2(n * alpha_tab[v + 3 * (as)]); \
  			q[(rb) + u + 3] = m + t; \
  			q[(rb) + u + 3 + (hk)] = m - t; \
  		} \
  	} while (0)
  
  /*
   * Output ranges:
   *   d0:   min=    0   max= 1020
   *   d1:   min=  -67   max= 4587
   *   d2:   min=-4335   max= 4335
   *   d3:   min=-4147   max=  507
   *   d4:   min= -510   max=  510
   *   d5:   min= -252   max= 4402
   *   d6:   min=-4335   max= 4335
   *   d7:   min=-4332   max=  322
   */
  #define FFT8(xb, xs, d)   do { \
  		s32 x0 = x[(xb)]; \
  		s32 x1 = x[(xb) + (xs)]; \
  		s32 x2 = x[(xb) + 2 * (xs)]; \
  		s32 x3 = x[(xb) + 3 * (xs)]; \
  		s32 a0 = x0 + x2; \
  		s32 a1 = x0 + (x2 << 4); \
  		s32 a2 = x0 - x2; \
  		s32 a3 = x0 - (x2 << 4); \
  		s32 b0 = x1 + x3; \
  		s32 b1 = REDS1((x1 << 2) + (x3 << 6)); \
  		s32 b2 = (x1 << 4) - (x3 << 4); \
  		s32 b3 = REDS1((x1 << 6) + (x3 << 2)); \
  		d ## 0 = a0 + b0; \
  		d ## 1 = a1 + b1; \
  		d ## 2 = a2 + b2; \
  		d ## 3 = a3 + b3; \
  		d ## 4 = a0 - b0; \
  		d ## 5 = a1 - b1; \
  		d ## 6 = a2 - b2; \
  		d ## 7 = a3 - b3; \
  	} while (0)
  
  /*
   * When k=16, we have alpha=2. Multiplication by alpha^i is then reduced
   * to some shifting.
   *
   * Output: within -591471..591723
   */
  #define FFT16(xb, xs, rb)   do { \
  		s32 d1_0, d1_1, d1_2, d1_3, d1_4, d1_5, d1_6, d1_7; \
  		s32 d2_0, d2_1, d2_2, d2_3, d2_4, d2_5, d2_6, d2_7; \
  		FFT8(xb, (xs) << 1, d1_); \
  		FFT8((xb) + (xs), (xs) << 1, d2_); \
  		q[(rb) +  0] = d1_0 + d2_0; \
  		q[(rb) +  1] = d1_1 + (d2_1 << 1); \
  		q[(rb) +  2] = d1_2 + (d2_2 << 2); \
  		q[(rb) +  3] = d1_3 + (d2_3 << 3); \
  		q[(rb) +  4] = d1_4 + (d2_4 << 4); \
  		q[(rb) +  5] = d1_5 + (d2_5 << 5); \
  		q[(rb) +  6] = d1_6 + (d2_6 << 6); \
  		q[(rb) +  7] = d1_7 + (d2_7 << 7); \
  		q[(rb) +  8] = d1_0 - d2_0; \
  		q[(rb) +  9] = d1_1 - (d2_1 << 1); \
  		q[(rb) + 10] = d1_2 - (d2_2 << 2); \
  		q[(rb) + 11] = d1_3 - (d2_3 << 3); \
  		q[(rb) + 12] = d1_4 - (d2_4 << 4); \
  		q[(rb) + 13] = d1_5 - (d2_5 << 5); \
  		q[(rb) + 14] = d1_6 - (d2_6 << 6); \
  		q[(rb) + 15] = d1_7 - (d2_7 << 7); \
  	} while (0)
  
  /*
   * Output range: |q| <= 1183446
   */
  #define FFT32(xb, xs, rb, id)   do { \
  		FFT16(xb, (xs) << 1, rb); \
  		FFT16((xb) + (xs), (xs) << 1, (rb) + 16); \
  		FFT_LOOP(rb, 16, 8, id); \
  	} while (0)
  
  /*
   * Output range: |q| <= 2366892
   */
  #define FFT64(xb, xs, rb, id)   do { \
  		FFT32(xb, (xs) << 1, rb, XCAT(id, a)); \
  		FFT32((xb) + (xs), (xs) << 1, (rb) + 32, XCAT(id, b)); \
  		FFT_LOOP(rb, 32, 4, id); \
  	} while (0)
  
  #if SPH_SMALL_FOOTPRINT_SIMD
  
  static void
  fft32(unsigned char *x, size_t xs, s32 *q)
  {
  	size_t xd;
  
  	xd = xs << 1;
  	FFT16(0, xd, 0);
  	FFT16(xs, xd, 16);
  	FFT_LOOP(0, 16, 8, label_);
  }
  
  #define FFT128(xb, xs, rb, id)   do { \
  		fft32(x + (xb) + ((xs) * 0), (xs) << 2, &q[(rb) +  0]); \
  		fft32(x + (xb) + ((xs) * 2), (xs) << 2, &q[(rb) + 32]); \
  		FFT_LOOP(rb, 32, 4, XCAT(id, aa)); \
  		fft32(x + (xb) + ((xs) * 1), (xs) << 2, &q[(rb) + 64]); \
  		fft32(x + (xb) + ((xs) * 3), (xs) << 2, &q[(rb) + 96]); \
  		FFT_LOOP((rb) + 64, 32, 4, XCAT(id, ab)); \
  		FFT_LOOP(rb, 64, 2, XCAT(id, a)); \
  	} while (0)
  
  #else
  
  /*
   * Output range: |q| <= 4733784
   */
  #define FFT128(xb, xs, rb, id)   do { \
  		FFT64(xb, (xs) << 1, rb, XCAT(id, a)); \
  		FFT64((xb) + (xs), (xs) << 1, (rb) + 64, XCAT(id, b)); \
  		FFT_LOOP(rb, 64, 2, id); \
  	} while (0)
  
  #endif
  
  /*
   * For SIMD-384 / SIMD-512, the fully unrolled FFT yields a compression
   * function which does not fit in the 32 kB L1 cache of a typical x86
   * Intel. We therefore add a function call layer at the FFT64 level.
   */
  
  static void
  fft64(unsigned char *x, size_t xs, s32 *q)
  {
  	size_t xd;
  
  	xd = xs << 1;
  	FFT32(0, xd, 0, label_a);
  	FFT32(xs, xd, 32, label_b);
  	FFT_LOOP(0, 32, 4, label_);
  }
  
  /*
   * Output range: |q| <= 9467568
   */
  #define FFT256(xb, xs, rb, id)   do { \
  		fft64(x + (xb) + ((xs) * 0), (xs) << 2, &q[(rb) +   0]); \
  		fft64(x + (xb) + ((xs) * 2), (xs) << 2, &q[(rb) +  64]); \
  		FFT_LOOP(rb, 64, 2, XCAT(id, aa)); \
  		fft64(x + (xb) + ((xs) * 1), (xs) << 2, &q[(rb) + 128]); \
  		fft64(x + (xb) + ((xs) * 3), (xs) << 2, &q[(rb) + 192]); \
  		FFT_LOOP((rb) + 128, 64, 2, XCAT(id, ab)); \
  		FFT_LOOP(rb, 128, 1, XCAT(id, a)); \
  	} while (0)
  
  /*
   * alpha^(127*i) mod 257
   */
  static const unsigned short yoff_s_n[] = {
  	  1,  98,  95,  58,  30, 113,  23, 198, 129,  49, 176,  29,
  	 15, 185, 140,  99, 193, 153,  88, 143, 136, 221,  70, 178,
  	225, 205,  44, 200,  68, 239,  35,  89, 241, 231,  22, 100,
  	 34, 248, 146, 173, 249, 244,  11,  50,  17, 124,  73, 215,
  	253, 122, 134,  25, 137,  62, 165, 236, 255,  61,  67, 141,
  	197,  31, 211, 118, 256, 159, 162, 199, 227, 144, 234,  59,
  	128, 208,  81, 228, 242,  72, 117, 158,  64, 104, 169, 114,
  	121,  36, 187,  79,  32,  52, 213,  57, 189,  18, 222, 168,
  	 16,  26, 235, 157, 223,   9, 111,  84,   8,  13, 246, 207,
  	240, 133, 184,  42,   4, 135, 123, 232, 120, 195,  92,  21,
  	  2, 196, 190, 116,  60, 226,  46, 139
  };
  
  /*
   * alpha^(127*i) + alpha^(125*i) mod 257
   */
  static const unsigned short yoff_s_f[] = {
  	  2, 156, 118, 107,  45, 212, 111, 162,  97, 249, 211,   3,
  	 49, 101, 151, 223, 189, 178, 253, 204,  76,  82, 232,  65,
  	 96, 176, 161,  47, 189,  61, 248, 107,   0, 131, 133, 113,
  	 17,  33,  12, 111, 251, 103,  57, 148,  47,  65, 249, 143,
  	189,   8, 204, 230, 205, 151, 187, 227, 247, 111, 140,   6,
  	 77,  10,  21, 149, 255, 101, 139, 150, 212,  45, 146,  95,
  	160,   8,  46, 254, 208, 156, 106,  34,  68,  79,   4,  53,
  	181, 175,  25, 192, 161,  81,  96, 210,  68, 196,   9, 150,
  	  0, 126, 124, 144, 240, 224, 245, 146,   6, 154, 200, 109,
  	210, 192,   8, 114,  68, 249,  53,  27,  52, 106,  70,  30,
  	 10, 146, 117, 251, 180, 247, 236, 108
  };
  
  /*
   * beta^(255*i) mod 257
   */
  static const unsigned short yoff_b_n[] = {
  	  1, 163,  98,  40,  95,  65,  58, 202,  30,   7, 113, 172,
  	 23, 151, 198, 149, 129, 210,  49,  20, 176, 161,  29, 101,
  	 15, 132, 185,  86, 140, 204,  99, 203, 193, 105, 153,  10,
  	 88, 209, 143, 179, 136,  66, 221,  43,  70, 102, 178, 230,
  	225, 181, 205,   5,  44, 233, 200, 218,  68,  33, 239, 150,
  	 35,  51,  89, 115, 241, 219, 231, 131,  22, 245, 100, 109,
  	 34, 145, 248,  75, 146, 154, 173, 186, 249, 238, 244, 194,
  	 11, 251,  50, 183,  17, 201, 124, 166,  73,  77, 215,  93,
  	253, 119, 122,  97, 134, 254,  25, 220, 137, 229,  62,  83,
  	165, 167, 236, 175, 255, 188,  61, 177,  67, 127, 141, 110,
  	197, 243,  31, 170, 211, 212, 118, 216, 256,  94, 159, 217,
  	162, 192, 199,  55, 227, 250, 144,  85, 234, 106,  59, 108,
  	128,  47, 208, 237,  81,  96, 228, 156, 242, 125,  72, 171,
  	117,  53, 158,  54,  64, 152, 104, 247, 169,  48, 114,  78,
  	121, 191,  36, 214, 187, 155,  79,  27,  32,  76,  52, 252,
  	213,  24,  57,  39, 189, 224,  18, 107, 222, 206, 168, 142,
  	 16,  38,  26, 126, 235,  12, 157, 148, 223, 112,   9, 182,
  	111, 103,  84,  71,   8,  19,  13,  63, 246,   6, 207,  74,
  	240,  56, 133,  91, 184, 180,  42, 164,   4, 138, 135, 160,
  	123,   3, 232,  37, 120,  28, 195, 174,  92,  90,  21,  82,
  	  2,  69, 196,  80, 190, 130, 116, 147,  60,  14, 226,  87,
  	 46,  45, 139,  41
  };
  
  /*
   * beta^(255*i) + beta^(253*i) mod 257
   */
  static const unsigned short yoff_b_f[] = {
  	  2, 203, 156,  47, 118, 214, 107, 106,  45,  93, 212,  20,
  	111,  73, 162, 251,  97, 215, 249,  53, 211,  19,   3,  89,
  	 49, 207, 101,  67, 151, 130, 223,  23, 189, 202, 178, 239,
  	253, 127, 204,  49,  76, 236,  82, 137, 232, 157,  65,  79,
  	 96, 161, 176, 130, 161,  30,  47,   9, 189, 247,  61, 226,
  	248,  90, 107,  64,   0,  88, 131, 243, 133,  59, 113, 115,
  	 17, 236,  33, 213,  12, 191, 111,  19, 251,  61, 103, 208,
  	 57,  35, 148, 248,  47, 116,  65, 119, 249, 178, 143,  40,
  	189, 129,   8, 163, 204, 227, 230, 196, 205, 122, 151,  45,
  	187,  19, 227,  72, 247, 125, 111, 121, 140, 220,   6, 107,
  	 77,  69,  10, 101,  21,  65, 149, 171, 255,  54, 101, 210,
  	139,  43, 150, 151, 212, 164,  45, 237, 146, 184,  95,   6,
  	160,  42,   8, 204,  46, 238, 254, 168, 208,  50, 156, 190,
  	106, 127,  34, 234,  68,  55,  79,  18,   4, 130,  53, 208,
  	181,  21, 175, 120,  25, 100, 192, 178, 161,  96,  81, 127,
  	 96, 227, 210, 248,  68,  10, 196,  31,   9, 167, 150, 193,
  	  0, 169, 126,  14, 124, 198, 144, 142, 240,  21, 224,  44,
  	245,  66, 146, 238,   6, 196, 154,  49, 200, 222, 109,   9,
  	210, 141, 192, 138,   8,  79, 114, 217,  68, 128, 249,  94,
  	 53,  30,  27,  61,  52, 135, 106, 212,  70, 238,  30, 185,
  	 10, 132, 146, 136, 117,  37, 251, 150, 180, 188, 247, 156,
  	236, 192, 108,  86
  };
  
  #define INNER(l, h, mm)   (((u32)((l) * (mm)) & 0xFFFFU) \
                            + ((u32)((h) * (mm)) << 16))
  
  #define W_SMALL(sb, o1, o2, mm) \
  	(INNER(q[8 * (sb) + 2 * 0 + o1], q[8 * (sb) + 2 * 0 + o2], mm), \
  	 INNER(q[8 * (sb) + 2 * 1 + o1], q[8 * (sb) + 2 * 1 + o2], mm), \
  	 INNER(q[8 * (sb) + 2 * 2 + o1], q[8 * (sb) + 2 * 2 + o2], mm), \
  	 INNER(q[8 * (sb) + 2 * 3 + o1], q[8 * (sb) + 2 * 3 + o2], mm)
  
  #define WS_0_0   W_SMALL( 4,    0,    1, 185)
  #define WS_0_1   W_SMALL( 6,    0,    1, 185)
  #define WS_0_2   W_SMALL( 0,    0,    1, 185)
  #define WS_0_3   W_SMALL( 2,    0,    1, 185)
  #define WS_0_4   W_SMALL( 7,    0,    1, 185)
  #define WS_0_5   W_SMALL( 5,    0,    1, 185)
  #define WS_0_6   W_SMALL( 3,    0,    1, 185)
  #define WS_0_7   W_SMALL( 1,    0,    1, 185)
  #define WS_1_0   W_SMALL(15,    0,    1, 185)
  #define WS_1_1   W_SMALL(11,    0,    1, 185)
  #define WS_1_2   W_SMALL(12,    0,    1, 185)
  #define WS_1_3   W_SMALL( 8,    0,    1, 185)
  #define WS_1_4   W_SMALL( 9,    0,    1, 185)
  #define WS_1_5   W_SMALL(13,    0,    1, 185)
  #define WS_1_6   W_SMALL(10,    0,    1, 185)
  #define WS_1_7   W_SMALL(14,    0,    1, 185)
  #define WS_2_0   W_SMALL(17, -128,  -64, 233)
  #define WS_2_1   W_SMALL(18, -128,  -64, 233)
  #define WS_2_2   W_SMALL(23, -128,  -64, 233)
  #define WS_2_3   W_SMALL(20, -128,  -64, 233)
  #define WS_2_4   W_SMALL(22, -128,  -64, 233)
  #define WS_2_5   W_SMALL(21, -128,  -64, 233)
  #define WS_2_6   W_SMALL(16, -128,  -64, 233)
  #define WS_2_7   W_SMALL(19, -128,  -64, 233)
  #define WS_3_0   W_SMALL(30, -191, -127, 233)
  #define WS_3_1   W_SMALL(24, -191, -127, 233)
  #define WS_3_2   W_SMALL(25, -191, -127, 233)
  #define WS_3_3   W_SMALL(31, -191, -127, 233)
  #define WS_3_4   W_SMALL(27, -191, -127, 233)
  #define WS_3_5   W_SMALL(29, -191, -127, 233)
  #define WS_3_6   W_SMALL(28, -191, -127, 233)
  #define WS_3_7   W_SMALL(26, -191, -127, 233)
  
  #define W_BIG(sb, o1, o2, mm) \
  	(INNER(q[16 * (sb) + 2 * 0 + o1], q[16 * (sb) + 2 * 0 + o2], mm), \
  	 INNER(q[16 * (sb) + 2 * 1 + o1], q[16 * (sb) + 2 * 1 + o2], mm), \
  	 INNER(q[16 * (sb) + 2 * 2 + o1], q[16 * (sb) + 2 * 2 + o2], mm), \
  	 INNER(q[16 * (sb) + 2 * 3 + o1], q[16 * (sb) + 2 * 3 + o2], mm), \
  	 INNER(q[16 * (sb) + 2 * 4 + o1], q[16 * (sb) + 2 * 4 + o2], mm), \
  	 INNER(q[16 * (sb) + 2 * 5 + o1], q[16 * (sb) + 2 * 5 + o2], mm), \
  	 INNER(q[16 * (sb) + 2 * 6 + o1], q[16 * (sb) + 2 * 6 + o2], mm), \
  	 INNER(q[16 * (sb) + 2 * 7 + o1], q[16 * (sb) + 2 * 7 + o2], mm)
  
  #define WB_0_0   W_BIG( 4,    0,    1, 185)
  #define WB_0_1   W_BIG( 6,    0,    1, 185)
  #define WB_0_2   W_BIG( 0,    0,    1, 185)
  #define WB_0_3   W_BIG( 2,    0,    1, 185)
  #define WB_0_4   W_BIG( 7,    0,    1, 185)
  #define WB_0_5   W_BIG( 5,    0,    1, 185)
  #define WB_0_6   W_BIG( 3,    0,    1, 185)
  #define WB_0_7   W_BIG( 1,    0,    1, 185)
  #define WB_1_0   W_BIG(15,    0,    1, 185)
  #define WB_1_1   W_BIG(11,    0,    1, 185)
  #define WB_1_2   W_BIG(12,    0,    1, 185)
  #define WB_1_3   W_BIG( 8,    0,    1, 185)
  #define WB_1_4   W_BIG( 9,    0,    1, 185)
  #define WB_1_5   W_BIG(13,    0,    1, 185)
  #define WB_1_6   W_BIG(10,    0,    1, 185)
  #define WB_1_7   W_BIG(14,    0,    1, 185)
  #define WB_2_0   W_BIG(17, -256, -128, 233)
  #define WB_2_1   W_BIG(18, -256, -128, 233)
  #define WB_2_2   W_BIG(23, -256, -128, 233)
  #define WB_2_3   W_BIG(20, -256, -128, 233)
  #define WB_2_4   W_BIG(22, -256, -128, 233)
  #define WB_2_5   W_BIG(21, -256, -128, 233)
  #define WB_2_6   W_BIG(16, -256, -128, 233)
  #define WB_2_7   W_BIG(19, -256, -128, 233)
  #define WB_3_0   W_BIG(30, -383, -255, 233)
  #define WB_3_1   W_BIG(24, -383, -255, 233)
  #define WB_3_2   W_BIG(25, -383, -255, 233)
  #define WB_3_3   W_BIG(31, -383, -255, 233)
  #define WB_3_4   W_BIG(27, -383, -255, 233)
  #define WB_3_5   W_BIG(29, -383, -255, 233)
  #define WB_3_6   W_BIG(28, -383, -255, 233)
  #define WB_3_7   W_BIG(26, -383, -255, 233)
  
  #define IF(x, y, z)    ((((y) ^ (z)) & (x)) ^ (z))
  #define MAJ(x, y, z)   (((x) & (y)) | (((x) | (y)) & (z)))
  
  #define PP4_0_0   1
  #define PP4_0_1   0
  #define PP4_0_2   3
  #define PP4_0_3   2
  #define PP4_1_0   2
  #define PP4_1_1   3
  #define PP4_1_2   0
  #define PP4_1_3   1
  #define PP4_2_0   3
  #define PP4_2_1   2
  #define PP4_2_2   1
  #define PP4_2_3   0
  
  #define PP8_0_0   1
  #define PP8_0_1   0
  #define PP8_0_2   3
  #define PP8_0_3   2
  #define PP8_0_4   5
  #define PP8_0_5   4
  #define PP8_0_6   7
  #define PP8_0_7   6
  
  #define PP8_1_0   6
  #define PP8_1_1   7
  #define PP8_1_2   4
  #define PP8_1_3   5
  #define PP8_1_4   2
  #define PP8_1_5   3
  #define PP8_1_6   0
  #define PP8_1_7   1
  
  #define PP8_2_0   2
  #define PP8_2_1   3
  #define PP8_2_2   0
  #define PP8_2_3   1
  #define PP8_2_4   6
  #define PP8_2_5   7
  #define PP8_2_6   4
  #define PP8_2_7   5
  
  #define PP8_3_0   3
  #define PP8_3_1   2
  #define PP8_3_2   1
  #define PP8_3_3   0
  #define PP8_3_4   7
  #define PP8_3_5   6
  #define PP8_3_6   5
  #define PP8_3_7   4
  
  #define PP8_4_0   5
  #define PP8_4_1   4
  #define PP8_4_2   7
  #define PP8_4_3   6
  #define PP8_4_4   1
  #define PP8_4_5   0
  #define PP8_4_6   3
  #define PP8_4_7   2
  
  #define PP8_5_0   7
  #define PP8_5_1   6
  #define PP8_5_2   5
  #define PP8_5_3   4
  #define PP8_5_4   3
  #define PP8_5_5   2
  #define PP8_5_6   1
  #define PP8_5_7   0
  
  #define PP8_6_0   4
  #define PP8_6_1   5
  #define PP8_6_2   6
  #define PP8_6_3   7
  #define PP8_6_4   0
  #define PP8_6_5   1
  #define PP8_6_6   2
  #define PP8_6_7   3
  
  #if SPH_SIMD_NOCOPY
  
  #define DECL_STATE_SMALL
  #define READ_STATE_SMALL(sc)
  #define WRITE_STATE_SMALL(sc)
  #define DECL_STATE_BIG
  #define READ_STATE_BIG(sc)
  #define WRITE_STATE_BIG(sc)
  
  #else
  
  #define DECL_STATE_SMALL   \
  	u32 A0, A1, A2, A3, B0, B1, B2, B3, C0, C1, C2, C3, D0, D1, D2, D3;
  
  #define READ_STATE_SMALL(sc)   do { \
  		A0 = (sc)->state[ 0]; \
  		A1 = (sc)->state[ 1]; \
  		A2 = (sc)->state[ 2]; \
  		A3 = (sc)->state[ 3]; \
  		B0 = (sc)->state[ 4]; \
  		B1 = (sc)->state[ 5]; \
  		B2 = (sc)->state[ 6]; \
  		B3 = (sc)->state[ 7]; \
  		C0 = (sc)->state[ 8]; \
  		C1 = (sc)->state[ 9]; \
  		C2 = (sc)->state[10]; \
  		C3 = (sc)->state[11]; \
  		D0 = (sc)->state[12]; \
  		D1 = (sc)->state[13]; \
  		D2 = (sc)->state[14]; \
  		D3 = (sc)->state[15]; \
  	} while (0)
  
  #define WRITE_STATE_SMALL(sc)   do { \
  		(sc)->state[ 0] = A0; \
  		(sc)->state[ 1] = A1; \
  		(sc)->state[ 2] = A2; \
  		(sc)->state[ 3] = A3; \
  		(sc)->state[ 4] = B0; \
  		(sc)->state[ 5] = B1; \
  		(sc)->state[ 6] = B2; \
  		(sc)->state[ 7] = B3; \
  		(sc)->state[ 8] = C0; \
  		(sc)->state[ 9] = C1; \
  		(sc)->state[10] = C2; \
  		(sc)->state[11] = C3; \
  		(sc)->state[12] = D0; \
  		(sc)->state[13] = D1; \
  		(sc)->state[14] = D2; \
  		(sc)->state[15] = D3; \
  	} while (0)
  
  #define DECL_STATE_BIG   \
  	u32 A0, A1, A2, A3, A4, A5, A6, A7; \
  	u32 B0, B1, B2, B3, B4, B5, B6, B7; \
  	u32 C0, C1, C2, C3, C4, C5, C6, C7; \
  	u32 D0, D1, D2, D3, D4, D5, D6, D7;
  
  #define READ_STATE_BIG(sc)   do { \
  		A0 = (sc)->state[ 0]; \
  		A1 = (sc)->state[ 1]; \
  		A2 = (sc)->state[ 2]; \
  		A3 = (sc)->state[ 3]; \
  		A4 = (sc)->state[ 4]; \
  		A5 = (sc)->state[ 5]; \
  		A6 = (sc)->state[ 6]; \
  		A7 = (sc)->state[ 7]; \
  		B0 = (sc)->state[ 8]; \
  		B1 = (sc)->state[ 9]; \
  		B2 = (sc)->state[10]; \
  		B3 = (sc)->state[11]; \
  		B4 = (sc)->state[12]; \
  		B5 = (sc)->state[13]; \
  		B6 = (sc)->state[14]; \
  		B7 = (sc)->state[15]; \
  		C0 = (sc)->state[16]; \
  		C1 = (sc)->state[17]; \
  		C2 = (sc)->state[18]; \
  		C3 = (sc)->state[19]; \
  		C4 = (sc)->state[20]; \
  		C5 = (sc)->state[21]; \
  		C6 = (sc)->state[22]; \
  		C7 = (sc)->state[23]; \
  		D0 = (sc)->state[24]; \
  		D1 = (sc)->state[25]; \
  		D2 = (sc)->state[26]; \
  		D3 = (sc)->state[27]; \
  		D4 = (sc)->state[28]; \
  		D5 = (sc)->state[29]; \
  		D6 = (sc)->state[30]; \
  		D7 = (sc)->state[31]; \
  	} while (0)
  
  #define WRITE_STATE_BIG(sc)   do { \
  		(sc)->state[ 0] = A0; \
  		(sc)->state[ 1] = A1; \
  		(sc)->state[ 2] = A2; \
  		(sc)->state[ 3] = A3; \
  		(sc)->state[ 4] = A4; \
  		(sc)->state[ 5] = A5; \
  		(sc)->state[ 6] = A6; \
  		(sc)->state[ 7] = A7; \
  		(sc)->state[ 8] = B0; \
  		(sc)->state[ 9] = B1; \
  		(sc)->state[10] = B2; \
  		(sc)->state[11] = B3; \
  		(sc)->state[12] = B4; \
  		(sc)->state[13] = B5; \
  		(sc)->state[14] = B6; \
  		(sc)->state[15] = B7; \
  		(sc)->state[16] = C0; \
  		(sc)->state[17] = C1; \
  		(sc)->state[18] = C2; \
  		(sc)->state[19] = C3; \
  		(sc)->state[20] = C4; \
  		(sc)->state[21] = C5; \
  		(sc)->state[22] = C6; \
  		(sc)->state[23] = C7; \
  		(sc)->state[24] = D0; \
  		(sc)->state[25] = D1; \
  		(sc)->state[26] = D2; \
  		(sc)->state[27] = D3; \
  		(sc)->state[28] = D4; \
  		(sc)->state[29] = D5; \
  		(sc)->state[30] = D6; \
  		(sc)->state[31] = D7; \
  	} while (0)
  
  #endif
  
  #define STEP_ELT(n, w, fun, s, ppb)   do { \
  		u32 tt = T32(D ## n + (w) + fun(A ## n, B ## n, C ## n)); \
  		A ## n = T32(ROL32(tt, s) + XCAT(tA, XCAT(ppb, n))); \
  		D ## n = C ## n; \
  		C ## n = B ## n; \
  		B ## n = tA ## n; \
  	} while (0)
  
  #define STEP_SMALL(w0, w1, w2, w3, fun, r, s, pp4b)   do { \
  		u32 tA0 = ROL32(A0, r); \
  		u32 tA1 = ROL32(A1, r); \
  		u32 tA2 = ROL32(A2, r); \
  		u32 tA3 = ROL32(A3, r); \
  		STEP_ELT(0, w0, fun, s, pp4b); \
  		STEP_ELT(1, w1, fun, s, pp4b); \
  		STEP_ELT(2, w2, fun, s, pp4b); \
  		STEP_ELT(3, w3, fun, s, pp4b); \
  	} while (0)
  
  #define STEP_BIG(w0, w1, w2, w3, w4, w5, w6, w7, fun, r, s, pp8b)   do { \
  		u32 tA0 = ROL32(A0, r); \
  		u32 tA1 = ROL32(A1, r); \
  		u32 tA2 = ROL32(A2, r); \
  		u32 tA3 = ROL32(A3, r); \
  		u32 tA4 = ROL32(A4, r); \
  		u32 tA5 = ROL32(A5, r); \
  		u32 tA6 = ROL32(A6, r); \
  		u32 tA7 = ROL32(A7, r); \
  		STEP_ELT(0, w0, fun, s, pp8b); \
  		STEP_ELT(1, w1, fun, s, pp8b); \
  		STEP_ELT(2, w2, fun, s, pp8b); \
  		STEP_ELT(3, w3, fun, s, pp8b); \
  		STEP_ELT(4, w4, fun, s, pp8b); \
  		STEP_ELT(5, w5, fun, s, pp8b); \
  		STEP_ELT(6, w6, fun, s, pp8b); \
  		STEP_ELT(7, w7, fun, s, pp8b); \
  	} while (0)
  
  #define M3_0_0   0_
  #define M3_1_0   1_
  #define M3_2_0   2_
  #define M3_3_0   0_
  #define M3_4_0   1_
  #define M3_5_0   2_
  #define M3_6_0   0_
  #define M3_7_0   1_
  
  #define M3_0_1   1_
  #define M3_1_1   2_
  #define M3_2_1   0_
  #define M3_3_1   1_
  #define M3_4_1   2_
  #define M3_5_1   0_
  #define M3_6_1   1_
  #define M3_7_1   2_
  
  #define M3_0_2   2_
  #define M3_1_2   0_
  #define M3_2_2   1_
  #define M3_3_2   2_
  #define M3_4_2   0_
  #define M3_5_2   1_
  #define M3_6_2   2_
  #define M3_7_2   0_
  
  #define STEP_SMALL_(w, fun, r, s, pp4b)   STEP_SMALL w, fun, r, s, pp4b)
  
  #define ONE_ROUND_SMALL(ri, isp, p0, p1, p2, p3)   do { \
  		STEP_SMALL_(WS_ ## ri ## 0, \
  			IF,  p0, p1, XCAT(PP4_, M3_0_ ## isp)); \
  		STEP_SMALL_(WS_ ## ri ## 1, \
  			IF,  p1, p2, XCAT(PP4_, M3_1_ ## isp)); \
  		STEP_SMALL_(WS_ ## ri ## 2, \
  			IF,  p2, p3, XCAT(PP4_, M3_2_ ## isp)); \
  		STEP_SMALL_(WS_ ## ri ## 3, \
  			IF,  p3, p0, XCAT(PP4_, M3_3_ ## isp)); \
  		STEP_SMALL_(WS_ ## ri ## 4, \
  			MAJ, p0, p1, XCAT(PP4_, M3_4_ ## isp)); \
  		STEP_SMALL_(WS_ ## ri ## 5, \
  			MAJ, p1, p2, XCAT(PP4_, M3_5_ ## isp)); \
  		STEP_SMALL_(WS_ ## ri ## 6, \
  			MAJ, p2, p3, XCAT(PP4_, M3_6_ ## isp)); \
  		STEP_SMALL_(WS_ ## ri ## 7, \
  			MAJ, p3, p0, XCAT(PP4_, M3_7_ ## isp)); \
  	} while (0)
  
  #define M7_0_0   0_
  #define M7_1_0   1_
  #define M7_2_0   2_
  #define M7_3_0   3_
  #define M7_4_0   4_
  #define M7_5_0   5_
  #define M7_6_0   6_
  #define M7_7_0   0_
  
  #define M7_0_1   1_
  #define M7_1_1   2_
  #define M7_2_1   3_
  #define M7_3_1   4_
  #define M7_4_1   5_
  #define M7_5_1   6_
  #define M7_6_1   0_
  #define M7_7_1   1_
  
  #define M7_0_2   2_
  #define M7_1_2   3_
  #define M7_2_2   4_
  #define M7_3_2   5_
  #define M7_4_2   6_
  #define M7_5_2   0_
  #define M7_6_2   1_
  #define M7_7_2   2_
  
  #define M7_0_3   3_
  #define M7_1_3   4_
  #define M7_2_3   5_
  #define M7_3_3   6_
  #define M7_4_3   0_
  #define M7_5_3   1_
  #define M7_6_3   2_
  #define M7_7_3   3_
  
  #define STEP_BIG_(w, fun, r, s, pp8b)   STEP_BIG w, fun, r, s, pp8b)
  
  #define ONE_ROUND_BIG(ri, isp, p0, p1, p2, p3)   do { \
  		STEP_BIG_(WB_ ## ri ## 0, \
  			IF,  p0, p1, XCAT(PP8_, M7_0_ ## isp)); \
  		STEP_BIG_(WB_ ## ri ## 1, \
  			IF,  p1, p2, XCAT(PP8_, M7_1_ ## isp)); \
  		STEP_BIG_(WB_ ## ri ## 2, \
  			IF,  p2, p3, XCAT(PP8_, M7_2_ ## isp)); \
  		STEP_BIG_(WB_ ## ri ## 3, \
  			IF,  p3, p0, XCAT(PP8_, M7_3_ ## isp)); \
  		STEP_BIG_(WB_ ## ri ## 4, \
  			MAJ, p0, p1, XCAT(PP8_, M7_4_ ## isp)); \
  		STEP_BIG_(WB_ ## ri ## 5, \
  			MAJ, p1, p2, XCAT(PP8_, M7_5_ ## isp)); \
  		STEP_BIG_(WB_ ## ri ## 6, \
  			MAJ, p2, p3, XCAT(PP8_, M7_6_ ## isp)); \
  		STEP_BIG_(WB_ ## ri ## 7, \
  			MAJ, p3, p0, XCAT(PP8_, M7_7_ ## isp)); \
  	} while (0)
  
  #if SPH_SMALL_FOOTPRINT_SIMD
  
  #define A0   state[ 0]
  #define A1   state[ 1]
  #define A2   state[ 2]
  #define A3   state[ 3]
  #define B0   state[ 4]
  #define B1   state[ 5]
  #define B2   state[ 6]
  #define B3   state[ 7]
  #define C0   state[ 8]
  #define C1   state[ 9]
  #define C2   state[10]
  #define C3   state[11]
  #define D0   state[12]
  #define D1   state[13]
  #define D2   state[14]
  #define D3   state[15]
  
  #define STEP2_ELT(n, w, fun, s, ppb)   do { \
  		u32 tt = T32(D ## n + (w) + fun(A ## n, B ## n, C ## n)); \
  		A ## n = T32(ROL32(tt, s) + tA[(ppb) ^ n]); \
  		D ## n = C ## n; \
  		C ## n = B ## n; \
  		B ## n = tA[n]; \
  	} while (0)
  
  #define STEP2_SMALL(w0, w1, w2, w3, fun, r, s, pp4b)   do { \
  		u32 tA[4]; \
  		tA[0] = ROL32(A0, r); \
  		tA[1] = ROL32(A1, r); \
  		tA[2] = ROL32(A2, r); \
  		tA[3] = ROL32(A3, r); \
  		STEP2_ELT(0, w0, fun, s, pp4b); \
  		STEP2_ELT(1, w1, fun, s, pp4b); \
  		STEP2_ELT(2, w2, fun, s, pp4b); \
  		STEP2_ELT(3, w3, fun, s, pp4b); \
  	} while (0)
  
  static void
  one_round_small(u32 *state, u32 *w, int isp, int p0, int p1, int p2, int p3)
  {
  	static const int pp4k[] = { 1, 2, 3, 1, 2, 3, 1, 2, 3, 1, 2 };
  
  	STEP2_SMALL(w[ 0], w[ 1], w[ 2], w[ 3], IF,  p0, p1, pp4k[isp + 0]);
  	STEP2_SMALL(w[ 4], w[ 5], w[ 6], w[ 7], IF,  p1, p2, pp4k[isp + 1]);
  	STEP2_SMALL(w[ 8], w[ 9], w[10], w[11], IF,  p2, p3, pp4k[isp + 2]);
  	STEP2_SMALL(w[12], w[13], w[14], w[15], IF,  p3, p0, pp4k[isp + 3]);
  	STEP2_SMALL(w[16], w[17], w[18], w[19], MAJ, p0, p1, pp4k[isp + 4]);
  	STEP2_SMALL(w[20], w[21], w[22], w[23], MAJ, p1, p2, pp4k[isp + 5]);
  	STEP2_SMALL(w[24], w[25], w[26], w[27], MAJ, p2, p3, pp4k[isp + 6]);
  	STEP2_SMALL(w[28], w[29], w[30], w[31], MAJ, p3, p0, pp4k[isp + 7]);
  }
  
  static void
  compress_small(sph_simd_small_context *sc, int last)
  {
  	unsigned char *x;
  	s32 q[128];
  	int i;
  	u32 w[32];
  	u32 state[16];
  	size_t u;
  
  	static const size_t wsp[32] = {
  		 4 << 3,  6 << 3,  0 << 3,  2 << 3,
  		 7 << 3,  5 << 3,  3 << 3,  1 << 3,
  		15 << 3, 11 << 3, 12 << 3,  8 << 3,
  		 9 << 3, 13 << 3, 10 << 3, 14 << 3,
  		17 << 3, 18 << 3, 23 << 3, 20 << 3,
  		22 << 3, 21 << 3, 16 << 3, 19 << 3,
  		30 << 3, 24 << 3, 25 << 3, 31 << 3,
  		27 << 3, 29 << 3, 28 << 3, 26 << 3
  	};
  
  	x = sc->buf;
  	FFT128(0, 1, 0, ll);
  	if (last) {
  		for (i = 0; i < 128; i ++) {
  			s32 tq;
  
  			tq = q[i] + yoff_s_f[i];
  			tq = REDS2(tq);
  			tq = REDS1(tq);
  			tq = REDS1(tq);
  			q[i] = (tq <= 128 ? tq : tq - 257);
  		}
  	} else {
  		for (i = 0; i < 128; i ++) {
  			s32 tq;
  
  			tq = q[i] + yoff_s_n[i];
  			tq = REDS2(tq);
  			tq = REDS1(tq);
  			tq = REDS1(tq);
  			q[i] = (tq <= 128 ? tq : tq - 257);
  		}
  	}
  
  	for (i = 0; i < 16; i += 4) {
  		state[i + 0] = sc->state[i + 0]
  			^ sph_dec32le_aligned(x + 4 * (i + 0));
  		state[i + 1] = sc->state[i + 1]
  			^ sph_dec32le_aligned(x + 4 * (i + 1));
  		state[i + 2] = sc->state[i + 2]
  			^ sph_dec32le_aligned(x + 4 * (i + 2));
  		state[i + 3] = sc->state[i + 3]
  			^ sph_dec32le_aligned(x + 4 * (i + 3));
  	}
  
  #define WSREAD(sb, o1, o2, mm)   do { \
  		for (u = 0; u < 32; u += 4) { \
  			size_t v = wsp[(u >> 2) + (sb)]; \
  			w[u + 0] = INNER(q[v + 2 * 0 + (o1)], \
  				q[v + 2 * 0 + (o2)], mm); \
  			w[u + 1] = INNER(q[v + 2 * 1 + (o1)], \
  				q[v + 2 * 1 + (o2)], mm); \
  			w[u + 2] = INNER(q[v + 2 * 2 + (o1)], \
  				q[v + 2 * 2 + (o2)], mm); \
  			w[u + 3] = INNER(q[v + 2 * 3 + (o1)], \
  				q[v + 2 * 3 + (o2)], mm); \
  		} \
  	} while (0)
  
  	WSREAD( 0,    0,    1, 185);
  	one_round_small(state, w, 0,  3, 23, 17, 27);
  	WSREAD( 8,    0,    1, 185);
  	one_round_small(state, w, 2, 28, 19, 22,  7);
  	WSREAD(16, -128,  -64, 233);
  	one_round_small(state, w, 1, 29,  9, 15,  5);
  	WSREAD(24, -191, -127, 233);
  	one_round_small(state, w, 0,  4, 13, 10, 25);
  
  #undef WSREAD
  
  	STEP_SMALL(sc->state[ 0], sc->state[ 1], sc->state[ 2], sc->state[ 3],
  		IF,  4, 13, PP4_2_);
  	STEP_SMALL(sc->state[ 4], sc->state[ 5], sc->state[ 6], sc->state[ 7],
  		IF, 13, 10, PP4_0_);
  	STEP_SMALL(sc->state[ 8], sc->state[ 9], sc->state[10], sc->state[11],
  		IF, 10, 25, PP4_1_);
  	STEP_SMALL(sc->state[12], sc->state[13], sc->state[14], sc->state[15],
  		IF, 25,  4, PP4_2_);
  
  	memcpy(sc->state, state, sizeof state);
  }
  
  #undef A0
  #undef A1
  #undef A2
  #undef A3
  #undef B0
  #undef B1
  #undef B2
  #undef B3
  #undef C0
  #undef C1
  #undef C2
  #undef C3
  #undef D0
  #undef D1
  #undef D2
  #undef D3
  
  #else
  
  #if SPH_SIMD_NOCOPY
  #define A0   (sc->state[ 0])
  #define A1   (sc->state[ 1])
  #define A2   (sc->state[ 2])
  #define A3   (sc->state[ 3])
  #define B0   (sc->state[ 4])
  #define B1   (sc->state[ 5])
  #define B2   (sc->state[ 6])
  #define B3   (sc->state[ 7])
  #define C0   (sc->state[ 8])
  #define C1   (sc->state[ 9])
  #define C2   (sc->state[10])
  #define C3   (sc->state[11])
  #define D0   (sc->state[12])
  #define D1   (sc->state[13])
  #define D2   (sc->state[14])
  #define D3   (sc->state[15])
  #endif
  
  static void
  compress_small(sph_simd_small_context *sc, int last)
  {
  	unsigned char *x;
  	s32 q[128];
  	int i;
  	DECL_STATE_SMALL
  #if SPH_SIMD_NOCOPY
  	sph_u32 saved[16];
  #endif
  
  #if SPH_SIMD_NOCOPY
  	memcpy(saved, sc->state, sizeof saved);
  #endif
  	x = sc->buf;
  	FFT128(0, 1, 0, ll);
  	if (last) {
  		for (i = 0; i < 128; i ++) {
  			s32 tq;
  
  			tq = q[i] + yoff_s_f[i];
  			tq = REDS2(tq);
  			tq = REDS1(tq);
  			tq = REDS1(tq);
  			q[i] = (tq <= 128 ? tq : tq - 257);
  		}
  	} else {
  		for (i = 0; i < 128; i ++) {
  			s32 tq;
  
  			tq = q[i] + yoff_s_n[i];
  			tq = REDS2(tq);
  			tq = REDS1(tq);
  			tq = REDS1(tq);
  			q[i] = (tq <= 128 ? tq : tq - 257);
  		}
  	}
  	READ_STATE_SMALL(sc);
  	A0 ^= sph_dec32le_aligned(x +  0);
  	A1 ^= sph_dec32le_aligned(x +  4);
  	A2 ^= sph_dec32le_aligned(x +  8);
  	A3 ^= sph_dec32le_aligned(x + 12);
  	B0 ^= sph_dec32le_aligned(x + 16);
  	B1 ^= sph_dec32le_aligned(x + 20);
  	B2 ^= sph_dec32le_aligned(x + 24);
  	B3 ^= sph_dec32le_aligned(x + 28);
  	C0 ^= sph_dec32le_aligned(x + 32);
  	C1 ^= sph_dec32le_aligned(x + 36);
  	C2 ^= sph_dec32le_aligned(x + 40);
  	C3 ^= sph_dec32le_aligned(x + 44);
  	D0 ^= sph_dec32le_aligned(x + 48);
  	D1 ^= sph_dec32le_aligned(x + 52);
  	D2 ^= sph_dec32le_aligned(x + 56);
  	D3 ^= sph_dec32le_aligned(x + 60);
  	ONE_ROUND_SMALL(0_, 0,  3, 23, 17, 27);
  	ONE_ROUND_SMALL(1_, 2, 28, 19, 22,  7);
  	ONE_ROUND_SMALL(2_, 1, 29,  9, 15,  5);
  	ONE_ROUND_SMALL(3_, 0,  4, 13, 10, 25);
  #if SPH_SIMD_NOCOPY
  	STEP_SMALL(saved[ 0], saved[ 1], saved[ 2], saved[ 3],
  		IF,  4, 13, PP4_2_);
  	STEP_SMALL(saved[ 4], saved[ 5], saved[ 6], saved[ 7],
  		IF, 13, 10, PP4_0_);
  	STEP_SMALL(saved[ 8], saved[ 9], saved[10], saved[11],
  		IF, 10, 25, PP4_1_);
  	STEP_SMALL(saved[12], saved[13], saved[14], saved[15],
  		IF, 25,  4, PP4_2_);
  #else
  	STEP_SMALL(sc->state[ 0], sc->state[ 1], sc->state[ 2], sc->state[ 3],
  		IF,  4, 13, PP4_2_);
  	STEP_SMALL(sc->state[ 4], sc->state[ 5], sc->state[ 6], sc->state[ 7],
  		IF, 13, 10, PP4_0_);
  	STEP_SMALL(sc->state[ 8], sc->state[ 9], sc->state[10], sc->state[11],
  		IF, 10, 25, PP4_1_);
  	STEP_SMALL(sc->state[12], sc->state[13], sc->state[14], sc->state[15],
  		IF, 25,  4, PP4_2_);
  	WRITE_STATE_SMALL(sc);
  #endif
  }
  
  #if SPH_SIMD_NOCOPY
  #undef A0
  #undef A1
  #undef A2
  #undef A3
  #undef B0
  #undef B1
  #undef B2
  #undef B3
  #undef C0
  #undef C1
  #undef C2
  #undef C3
  #undef D0
  #undef D1
  #undef D2
  #undef D3
  #endif
  
  #endif
  
  #if SPH_SMALL_FOOTPRINT_SIMD
  
  #define A0   state[ 0]
  #define A1   state[ 1]
  #define A2   state[ 2]
  #define A3   state[ 3]
  #define A4   state[ 4]
  #define A5   state[ 5]
  #define A6   state[ 6]
  #define A7   state[ 7]
  #define B0   state[ 8]
  #define B1   state[ 9]
  #define B2   state[10]
  #define B3   state[11]
  #define B4   state[12]
  #define B5   state[13]
  #define B6   state[14]
  #define B7   state[15]
  #define C0   state[16]
  #define C1   state[17]
  #define C2   state[18]
  #define C3   state[19]
  #define C4   state[20]
  #define C5   state[21]
  #define C6   state[22]
  #define C7   state[23]
  #define D0   state[24]
  #define D1   state[25]
  #define D2   state[26]
  #define D3   state[27]
  #define D4   state[28]
  #define D5   state[29]
  #define D6   state[30]
  #define D7   state[31]
  
  /*
   * Not needed -- already defined for SIMD-224 / SIMD-256
   *
  #define STEP2_ELT(n, w, fun, s, ppb)   do { \
  		u32 tt = T32(D ## n + (w) + fun(A ## n, B ## n, C ## n)); \
  		A ## n = T32(ROL32(tt, s) + tA[(ppb) ^ n]); \
  		D ## n = C ## n; \
  		C ## n = B ## n; \
  		B ## n = tA[n]; \
  	} while (0)
   */
  
  #define STEP2_BIG(w0, w1, w2, w3, w4, w5, w6, w7, fun, r, s, pp8b)   do { \
  		u32 tA[8]; \
  		tA[0] = ROL32(A0, r); \
  		tA[1] = ROL32(A1, r); \
  		tA[2] = ROL32(A2, r); \
  		tA[3] = ROL32(A3, r); \
  		tA[4] = ROL32(A4, r); \
  		tA[5] = ROL32(A5, r); \
  		tA[6] = ROL32(A6, r); \
  		tA[7] = ROL32(A7, r); \
  		STEP2_ELT(0, w0, fun, s, pp8b); \
  		STEP2_ELT(1, w1, fun, s, pp8b); \
  		STEP2_ELT(2, w2, fun, s, pp8b); \
  		STEP2_ELT(3, w3, fun, s, pp8b); \
  		STEP2_ELT(4, w4, fun, s, pp8b); \
  		STEP2_ELT(5, w5, fun, s, pp8b); \
  		STEP2_ELT(6, w6, fun, s, pp8b); \
  		STEP2_ELT(7, w7, fun, s, pp8b); \
  	} while (0)
  
  static void
  one_round_big(u32 *state, u32 *w, int isp, int p0, int p1, int p2, int p3)
  {
  	static const int pp8k[] = { 1, 6, 2, 3, 5, 7, 4, 1, 6, 2, 3 };
  
  	STEP2_BIG(w[ 0], w[ 1], w[ 2], w[ 3], w[ 4], w[ 5], w[ 6], w[ 7],
  		IF,  p0, p1, pp8k[isp + 0]);
  	STEP2_BIG(w[ 8], w[ 9], w[10], w[11], w[12], w[13], w[14], w[15],
  		IF,  p1, p2, pp8k[isp + 1]);
  	STEP2_BIG(w[16], w[17], w[18], w[19], w[20], w[21], w[22], w[23],
  		IF,  p2, p3, pp8k[isp + 2]);
  	STEP2_BIG(w[24], w[25], w[26], w[27], w[28], w[29], w[30], w[31],
  		IF,  p3, p0, pp8k[isp + 3]);
  	STEP2_BIG(w[32], w[33], w[34], w[35], w[36], w[37], w[38], w[39],
  		MAJ, p0, p1, pp8k[isp + 4]);
  	STEP2_BIG(w[40], w[41], w[42], w[43], w[44], w[45], w[46], w[47],
  		MAJ, p1, p2, pp8k[isp + 5]);
  	STEP2_BIG(w[48], w[49], w[50], w[51], w[52], w[53], w[54], w[55],
  		MAJ, p2, p3, pp8k[isp + 6]);
  	STEP2_BIG(w[56], w[57], w[58], w[59], w[60], w[61], w[62], w[63],
  		MAJ, p3, p0, pp8k[isp + 7]);
  }
  
  static void
  compress_big(sph_simd_big_context *sc, int last)
  {
  	unsigned char *x;
  	s32 q[256];
  	int i;
  	u32 w[64];
  	u32 state[32];
  	size_t u;
  
  	static const size_t wbp[32] = {
  		 4 << 4,  6 << 4,  0 << 4,  2 << 4,
  		 7 << 4,  5 << 4,  3 << 4,  1 << 4,
  		15 << 4, 11 << 4, 12 << 4,  8 << 4,
  		 9 << 4, 13 << 4, 10 << 4, 14 << 4,
  		17 << 4, 18 << 4, 23 << 4, 20 << 4,
  		22 << 4, 21 << 4, 16 << 4, 19 << 4,
  		30 << 4, 24 << 4, 25 << 4, 31 << 4,
  		27 << 4, 29 << 4, 28 << 4, 26 << 4
  	};
  
  	x = sc->buf;
  	FFT256(0, 1, 0, ll);
  	if (last) {
  		for (i = 0; i < 256; i ++) {
  			s32 tq;
  
  			tq = q[i] + yoff_b_f[i];
  			tq = REDS2(tq);
  			tq = REDS1(tq);
  			tq = REDS1(tq);
  			q[i] = (tq <= 128 ? tq : tq - 257);
  		}
  	} else {
  		for (i = 0; i < 256; i ++) {
  			s32 tq;
  
  			tq = q[i] + yoff_b_n[i];
  			tq = REDS2(tq);
  			tq = REDS1(tq);
  			tq = REDS1(tq);
  			q[i] = (tq <= 128 ? tq : tq - 257);
  		}
  	}
  
  	for (i = 0; i < 32; i += 8) {
  		state[i + 0] = sc->state[i + 0]
  			^ sph_dec32le_aligned(x + 4 * (i + 0));
  		state[i + 1] = sc->state[i + 1]
  			^ sph_dec32le_aligned(x + 4 * (i + 1));
  		state[i + 2] = sc->state[i + 2]
  			^ sph_dec32le_aligned(x + 4 * (i + 2));
  		state[i + 3] = sc->state[i + 3]
  			^ sph_dec32le_aligned(x + 4 * (i + 3));
  		state[i + 4] = sc->state[i + 4]
  			^ sph_dec32le_aligned(x + 4 * (i + 4));
  		state[i + 5] = sc->state[i + 5]
  			^ sph_dec32le_aligned(x + 4 * (i + 5));
  		state[i + 6] = sc->state[i + 6]
  			^ sph_dec32le_aligned(x + 4 * (i + 6));
  		state[i + 7] = sc->state[i + 7]
  			^ sph_dec32le_aligned(x + 4 * (i + 7));
  	}
  
  #define WBREAD(sb, o1, o2, mm)   do { \
  		for (u = 0; u < 64; u += 8) { \
  			size_t v = wbp[(u >> 3) + (sb)]; \
  			w[u + 0] = INNER(q[v + 2 * 0 + (o1)], \
  				q[v + 2 * 0 + (o2)], mm); \
  			w[u + 1] = INNER(q[v + 2 * 1 + (o1)], \
  				q[v + 2 * 1 + (o2)], mm); \
  			w[u + 2] = INNER(q[v + 2 * 2 + (o1)], \
  				q[v + 2 * 2 + (o2)], mm); \
  			w[u + 3] = INNER(q[v + 2 * 3 + (o1)], \
  				q[v + 2 * 3 + (o2)], mm); \
  			w[u + 4] = INNER(q[v + 2 * 4 + (o1)], \
  				q[v + 2 * 4 + (o2)], mm); \
  			w[u + 5] = INNER(q[v + 2 * 5 + (o1)], \
  				q[v + 2 * 5 + (o2)], mm); \
  			w[u + 6] = INNER(q[v + 2 * 6 + (o1)], \
  				q[v + 2 * 6 + (o2)], mm); \
  			w[u + 7] = INNER(q[v + 2 * 7 + (o1)], \
  				q[v + 2 * 7 + (o2)], mm); \
  		} \
  	} while (0)
  
  	WBREAD( 0,    0,    1, 185);
  	one_round_big(state, w, 0,  3, 23, 17, 27);
  	WBREAD( 8,    0,    1, 185);
  	one_round_big(state, w, 1, 28, 19, 22,  7);
  	WBREAD(16, -256, -128, 233);
  	one_round_big(state, w, 2, 29,  9, 15,  5);
  	WBREAD(24, -383, -255, 233);
  	one_round_big(state, w, 3,  4, 13, 10, 25);
  
  #undef WBREAD
  
  	STEP_BIG(
  		sc->state[ 0], sc->state[ 1], sc->state[ 2], sc->state[ 3],
  		sc->state[ 4], sc->state[ 5], sc->state[ 6], sc->state[ 7],
  		IF,  4, 13, PP8_4_);
  	STEP_BIG(
  		sc->state[ 8], sc->state[ 9], sc->state[10], sc->state[11],
  		sc->state[12], sc->state[13], sc->state[14], sc->state[15],
  		IF, 13, 10, PP8_5_);
  	STEP_BIG(
  		sc->state[16], sc->state[17], sc->state[18], sc->state[19],
  		sc->state[20], sc->state[21], sc->state[22], sc->state[23],
  		IF, 10, 25, PP8_6_);
  	STEP_BIG(
  		sc->state[24], sc->state[25], sc->state[26], sc->state[27],
  		sc->state[28], sc->state[29], sc->state[30], sc->state[31],
  		IF, 25,  4, PP8_0_);
  
  	memcpy(sc->state, state, sizeof state);
  }
  
  #undef A0
  #undef A1
  #undef A2
  #undef A3
  #undef A4
  #undef A5
  #undef A6
  #undef A7
  #undef B0
  #undef B1
  #undef B2
  #undef B3
  #undef B4
  #undef B5
  #undef B6
  #undef B7
  #undef C0
  #undef C1
  #undef C2
  #undef C3
  #undef C4
  #undef C5
  #undef C6
  #undef C7
  #undef D0
  #undef D1
  #undef D2
  #undef D3
  #undef D4
  #undef D5
  #undef D6
  #undef D7
  
  #else
  
  #if SPH_SIMD_NOCOPY
  #define A0   (sc->state[ 0])
  #define A1   (sc->state[ 1])
  #define A2   (sc->state[ 2])
  #define A3   (sc->state[ 3])
  #define A4   (sc->state[ 4])
  #define A5   (sc->state[ 5])
  #define A6   (sc->state[ 6])
  #define A7   (sc->state[ 7])
  #define B0   (sc->state[ 8])
  #define B1   (sc->state[ 9])
  #define B2   (sc->state[10])
  #define B3   (sc->state[11])
  #define B4   (sc->state[12])
  #define B5   (sc->state[13])
  #define B6   (sc->state[14])
  #define B7   (sc->state[15])
  #define C0   (sc->state[16])
  #define C1   (sc->state[17])
  #define C2   (sc->state[18])
  #define C3   (sc->state[19])
  #define C4   (sc->state[20])
  #define C5   (sc->state[21])
  #define C6   (sc->state[22])
  #define C7   (sc->state[23])
  #define D0   (sc->state[24])
  #define D1   (sc->state[25])
  #define D2   (sc->state[26])
  #define D3   (sc->state[27])
  #define D4   (sc->state[28])
  #define D5   (sc->state[29])
  #define D6   (sc->state[30])
  #define D7   (sc->state[31])
  #endif
  
  static void
  compress_big(sph_simd_big_context *sc, int last)
  {
  	unsigned char *x;
  	s32 q[256];
  	int i;
  	DECL_STATE_BIG
  #if SPH_SIMD_NOCOPY
  	sph_u32 saved[32];
  #endif
  
  #if SPH_SIMD_NOCOPY
  	memcpy(saved, sc->state, sizeof saved);
  #endif
  
  	x = sc->buf;
  	FFT256(0, 1, 0, ll);
  	if (last) {
  		for (i = 0; i < 256; i ++) {
  			s32 tq;
  
  			tq = q[i] + yoff_b_f[i];
  			tq = REDS2(tq);
  			tq = REDS1(tq);
  			tq = REDS1(tq);
  			q[i] = (tq <= 128 ? tq : tq - 257);
  		}
  	} else {
  		for (i = 0; i < 256; i ++) {
  			s32 tq;
  
  			tq = q[i] + yoff_b_n[i];
  			tq = REDS2(tq);
  			tq = REDS1(tq);
  			tq = REDS1(tq);
  			q[i] = (tq <= 128 ? tq : tq - 257);
  		}
  	}
  	READ_STATE_BIG(sc);
  	A0 ^= sph_dec32le_aligned(x +   0);
  	A1 ^= sph_dec32le_aligned(x +   4);
  	A2 ^= sph_dec32le_aligned(x +   8);
  	A3 ^= sph_dec32le_aligned(x +  12);
  	A4 ^= sph_dec32le_aligned(x +  16);
  	A5 ^= sph_dec32le_aligned(x +  20);
  	A6 ^= sph_dec32le_aligned(x +  24);
  	A7 ^= sph_dec32le_aligned(x +  28);
  	B0 ^= sph_dec32le_aligned(x +  32);
  	B1 ^= sph_dec32le_aligned(x +  36);
  	B2 ^= sph_dec32le_aligned(x +  40);
  	B3 ^= sph_dec32le_aligned(x +  44);
  	B4 ^= sph_dec32le_aligned(x +  48);
  	B5 ^= sph_dec32le_aligned(x +  52);
  	B6 ^= sph_dec32le_aligned(x +  56);
  	B7 ^= sph_dec32le_aligned(x +  60);
  	C0 ^= sph_dec32le_aligned(x +  64);
  	C1 ^= sph_dec32le_aligned(x +  68);
  	C2 ^= sph_dec32le_aligned(x +  72);
  	C3 ^= sph_dec32le_aligned(x +  76);
  	C4 ^= sph_dec32le_aligned(x +  80);
  	C5 ^= sph_dec32le_aligned(x +  84);
  	C6 ^= sph_dec32le_aligned(x +  88);
  	C7 ^= sph_dec32le_aligned(x +  92);
  	D0 ^= sph_dec32le_aligned(x +  96);
  	D1 ^= sph_dec32le_aligned(x + 100);
  	D2 ^= sph_dec32le_aligned(x + 104);
  	D3 ^= sph_dec32le_aligned(x + 108);
  	D4 ^= sph_dec32le_aligned(x + 112);
  	D5 ^= sph_dec32le_aligned(x + 116);
  	D6 ^= sph_dec32le_aligned(x + 120);
  	D7 ^= sph_dec32le_aligned(x + 124);
  
  	ONE_ROUND_BIG(0_, 0,  3, 23, 17, 27);
  	ONE_ROUND_BIG(1_, 1, 28, 19, 22,  7);
  	ONE_ROUND_BIG(2_, 2, 29,  9, 15,  5);
  	ONE_ROUND_BIG(3_, 3,  4, 13, 10, 25);
  #if SPH_SIMD_NOCOPY
  	STEP_BIG(
  		saved[ 0], saved[ 1], saved[ 2], saved[ 3],
  		saved[ 4], saved[ 5], saved[ 6], saved[ 7],
  		IF,  4, 13, PP8_4_);
  	STEP_BIG(
  		saved[ 8], saved[ 9], saved[10], saved[11],
  		saved[12], saved[13], saved[14], saved[15],
  		IF, 13, 10, PP8_5_);
  	STEP_BIG(
  		saved[16], saved[17], saved[18], saved[19],
  		saved[20], saved[21], saved[22], saved[23],
  		IF, 10, 25, PP8_6_);
  	STEP_BIG(
  		saved[24], saved[25], saved[26], saved[27],
  		saved[28], saved[29], saved[30], saved[31],
  		IF, 25,  4, PP8_0_);
  #else
  	STEP_BIG(
  		sc->state[ 0], sc->state[ 1], sc->state[ 2], sc->state[ 3],
  		sc->state[ 4], sc->state[ 5], sc->state[ 6], sc->state[ 7],
  		IF,  4, 13, PP8_4_);
  	STEP_BIG(
  		sc->state[ 8], sc->state[ 9], sc->state[10], sc->state[11],
  		sc->state[12], sc->state[13], sc->state[14], sc->state[15],
  		IF, 13, 10, PP8_5_);
  	STEP_BIG(
  		sc->state[16], sc->state[17], sc->state[18], sc->state[19],
  		sc->state[20], sc->state[21], sc->state[22], sc->state[23],
  		IF, 10, 25, PP8_6_);
  	STEP_BIG(
  		sc->state[24], sc->state[25], sc->state[26], sc->state[27],
  		sc->state[28], sc->state[29], sc->state[30], sc->state[31],
  		IF, 25,  4, PP8_0_);
  	WRITE_STATE_BIG(sc);
  #endif
  }
  
  #if SPH_SIMD_NOCOPY
  #undef A0
  #undef A1
  #undef A2
  #undef A3
  #undef A4
  #undef A5
  #undef A6
  #undef A7
  #undef B0
  #undef B1
  #undef B2
  #undef B3
  #undef B4
  #undef B5
  #undef B6
  #undef B7
  #undef C0
  #undef C1
  #undef C2
  #undef C3
  #undef C4
  #undef C5
  #undef C6
  #undef C7
  #undef D0
  #undef D1
  #undef D2
  #undef D3
  #undef D4
  #undef D5
  #undef D6
  #undef D7
  #endif
  
  #endif
  
  static const u32 IV224[] = {
  	C32(0x33586E9F), C32(0x12FFF033), C32(0xB2D9F64D), C32(0x6F8FEA53),
  	C32(0xDE943106), C32(0x2742E439), C32(0x4FBAB5AC), C32(0x62B9FF96),
  	C32(0x22E7B0AF), C32(0xC862B3A8), C32(0x33E00CDC), C32(0x236B86A6),
  	C32(0xF64AE77C), C32(0xFA373B76), C32(0x7DC1EE5B), C32(0x7FB29CE8)
  };
  
  static const u32 IV256[] = {
  	C32(0x4D567983), C32(0x07190BA9), C32(0x8474577B), C32(0x39D726E9),
  	C32(0xAAF3D925), C32(0x3EE20B03), C32(0xAFD5E751), C32(0xC96006D3),
  	C32(0xC2C2BA14), C32(0x49B3BCB4), C32(0xF67CAF46), C32(0x668626C9),
  	C32(0xE2EAA8D2), C32(0x1FF47833), C32(0xD0C661A5), C32(0x55693DE1)
  };
  
  static const u32 IV384[] = {
  	C32(0x8A36EEBC), C32(0x94A3BD90), C32(0xD1537B83), C32(0xB25B070B),
  	C32(0xF463F1B5), C32(0xB6F81E20), C32(0x0055C339), C32(0xB4D144D1),
  	C32(0x7360CA61), C32(0x18361A03), C32(0x17DCB4B9), C32(0x3414C45A),
  	C32(0xA699A9D2), C32(0xE39E9664), C32(0x468BFE77), C32(0x51D062F8),
  	C32(0xB9E3BFE8), C32(0x63BECE2A), C32(0x8FE506B9), C32(0xF8CC4AC2),
  	C32(0x7AE11542), C32(0xB1AADDA1), C32(0x64B06794), C32(0x28D2F462),
  	C32(0xE64071EC), C32(0x1DEB91A8), C32(0x8AC8DB23), C32(0x3F782AB5),
  	C32(0x039B5CB8), C32(0x71DDD962), C32(0xFADE2CEA), C32(0x1416DF71)
  };
  
  static const u32 IV512[] = {
  	C32(0x0BA16B95), C32(0x72F999AD), C32(0x9FECC2AE), C32(0xBA3264FC),
  	C32(0x5E894929), C32(0x8E9F30E5), C32(0x2F1DAA37), C32(0xF0F2C558),
  	C32(0xAC506643), C32(0xA90635A5), C32(0xE25B878B), C32(0xAAB7878F),
  	C32(0x88817F7A), C32(0x0A02892B), C32(0x559A7550), C32(0x598F657E),
  	C32(0x7EEF60A1), C32(0x6B70E3E8), C32(0x9C1714D1), C32(0xB958E2A8),
  	C32(0xAB02675E), C32(0xED1C014F), C32(0xCD8D65BB), C32(0xFDB7A257),
  	C32(0x09254899), C32(0xD699C7BC), C32(0x9019B6DC), C32(0x2B9022E4),
  	C32(0x8FA14956), C32(0x21BF9BD3), C32(0xB94D0943), C32(0x6FFDDC22)
  };
  
  static void
  init_small(void *cc, const u32 *iv)
  {
  	sph_simd_small_context *sc;
  
  	sc = cc;
  	memcpy(sc->state, iv, sizeof sc->state);
  	sc->count_low = sc->count_high = 0;
  	sc->ptr = 0;
  }
  
  static void
  init_big(void *cc, const u32 *iv)
  {
  	sph_simd_big_context *sc;
  
  	sc = cc;
  	memcpy(sc->state, iv, sizeof sc->state);
  	sc->count_low = sc->count_high = 0;
  	sc->ptr = 0;
  }
  
  static void
  update_small(void *cc, const void *data, size_t len)
  {
  	sph_simd_small_context *sc;
  
  	sc = cc;
  	while (len > 0) {
  		size_t clen;
  
  		clen = (sizeof sc->buf) - sc->ptr;
  		if (clen > len)
  			clen = len;
  		memcpy(sc->buf + sc->ptr, data, clen);
  		data = (const unsigned char *)data + clen;
  		len -= clen;
  		if ((sc->ptr += clen) == sizeof sc->buf) {
  			compress_small(sc, 0);
  			sc->ptr = 0;
  			sc->count_low = T32(sc->count_low + 1);
  			if (sc->count_low == 0)
  				sc->count_high ++;
  		}
  	}
  }
  
  static void
  update_big(void *cc, const void *data, size_t len)
  {
  	sph_simd_big_context *sc;
  
  	sc = cc;
  	while (len > 0) {
  		size_t clen;
  
  		clen = (sizeof sc->buf) - sc->ptr;
  		if (clen > len)
  			clen = len;
  		memcpy(sc->buf + sc->ptr, data, clen);
  		data = (const unsigned char *)data + clen;
  		len -= clen;
  		if ((sc->ptr += clen) == sizeof sc->buf) {
  			compress_big(sc, 0);
  			sc->ptr = 0;
  			sc->count_low = T32(sc->count_low + 1);
  			if (sc->count_low == 0)
  				sc->count_high ++;
  		}
  	}
  }
  
  static void
  encode_count_small(unsigned char *dst,
  	u32 low, u32 high, size_t ptr, unsigned n)
  {
  	low = T32(low << 9);
  	high = T32(high << 9) + (low >> 23);
  	low += (ptr << 3) + n;
  	sph_enc32le(dst, low);
  	sph_enc32le(dst + 4, high);
  }
  
  static void
  encode_count_big(unsigned char *dst,
  	u32 low, u32 high, size_t ptr, unsigned n)
  {
  	low = T32(low << 10);
  	high = T32(high << 10) + (low >> 22);
  	low += (ptr << 3) + n;
  	sph_enc32le(dst, low);
  	sph_enc32le(dst + 4, high);
  }
  
  static void
  finalize_small(void *cc, unsigned ub, unsigned n, void *dst, size_t dst_len)
  {
  	sph_simd_small_context *sc;
  	unsigned char *d;
  	size_t u;
  
  	sc = cc;
  	if (sc->ptr > 0 || n > 0) {
  		memset(sc->buf + sc->ptr, 0,
  			(sizeof sc->buf) - sc->ptr);
  		sc->buf[sc->ptr] = ub & (0xFF << (8 - n));
  		compress_small(sc, 0);
  	}
  	memset(sc->buf, 0, sizeof sc->buf);
  	encode_count_small(sc->buf, sc->count_low, sc->count_high, sc->ptr, n);
  	compress_small(sc, 1);
  	d = dst;
  	for (d = dst, u = 0; u < dst_len; u ++)
  		sph_enc32le(d + (u << 2), sc->state[u]);
  }
  
  static void
  finalize_big(void *cc, unsigned ub, unsigned n, void *dst, size_t dst_len)
  {
  	sph_simd_big_context *sc;
  	unsigned char *d;
  	size_t u;
  
  	sc = cc;
  	if (sc->ptr > 0 || n > 0) {
  		memset(sc->buf + sc->ptr, 0,
  			(sizeof sc->buf) - sc->ptr);
  		sc->buf[sc->ptr] = ub & (0xFF << (8 - n));
  		compress_big(sc, 0);
  	}
  	memset(sc->buf, 0, sizeof sc->buf);
  	encode_count_big(sc->buf, sc->count_low, sc->count_high, sc->ptr, n);
  	compress_big(sc, 1);
  	d = dst;
  	for (d = dst, u = 0; u < dst_len; u ++)
  		sph_enc32le(d + (u << 2), sc->state[u]);
  }
  
  void
  sph_simd224_init(void *cc)
  {
  	init_small(cc, IV224);
  }
  
  void
  sph_simd224(void *cc, const void *data, size_t len)
  {
  	update_small(cc, data, len);
  }
  
  void
  sph_simd224_close(void *cc, void *dst)
  {
  	sph_simd224_addbits_and_close(cc, 0, 0, dst);
  }
  
  void
  sph_simd224_addbits_and_close(void *cc, unsigned ub, unsigned n, void *dst)
  {
  	finalize_small(cc, ub, n, dst, 7);
  	sph_simd224_init(cc);
  }
  
  void
  sph_simd256_init(void *cc)
  {
  	init_small(cc, IV256);
  }
  
  void
  sph_simd256(void *cc, const void *data, size_t len)
  {
  	update_small(cc, data, len);
  }
  
  void
  sph_simd256_close(void *cc, void *dst)
  {
  	sph_simd256_addbits_and_close(cc, 0, 0, dst);
  }
  
  void
  sph_simd256_addbits_and_close(void *cc, unsigned ub, unsigned n, void *dst)
  {
  	finalize_small(cc, ub, n, dst, 8);
  	sph_simd256_init(cc);
  }
  
  void
  sph_simd384_init(void *cc)
  {
  	init_big(cc, IV384);
  }
  
  void
  sph_simd384(void *cc, const void *data, size_t len)
  {
  	update_big(cc, data, len);
  }
  
  void
  sph_simd384_close(void *cc, void *dst)
  {
  	sph_simd384_addbits_and_close(cc, 0, 0, dst);
  }
  
  void
  sph_simd384_addbits_and_close(void *cc, unsigned ub, unsigned n, void *dst)
  {
  	finalize_big(cc, ub, n, dst, 12);
  	sph_simd384_init(cc);
  }
  
  void
  sph_simd512_init(void *cc)
  {
  	init_big(cc, IV512);
  }
  
  void
  sph_simd512(void *cc, const void *data, size_t len)
  {
  	update_big(cc, data, len);
  }
  
  void
  sph_simd512_close(void *cc, void *dst)
  {
  	sph_simd512_addbits_and_close(cc, 0, 0, dst);
  }
  
  void
  sph_simd512_addbits_and_close(void *cc, unsigned ub, unsigned n, void *dst)
  {
  	finalize_big(cc, ub, n, dst, 16);
  	sph_simd512_init(cc);
  }
  #ifdef __cplusplus
  }
  #endif