text: #!/usr/bin/env python3
  #
  # To the extent possible under law, Yawning Angel has waived all copyright
  # and related or neighboring rights to chacha20, using the Creative
  # Commons "CC0" public domain dedication. See LICENSE or
  # <http://creativecommons.org/publicdomain/zero/1.0/> for full details.
  
  #
  # cgo sucks.  Plan 9 assembly sucks.  Real languages have SIMD intrinsics.
  # The least terrible/retarded option is to use a Python code generator, so
  # that's what I did.
  #
  # Code based on Ted Krovetz's vec128 C implementation, with corrections
  # to use a 64 bit counter instead of 32 bit, and to allow unaligned input and
  # output pointers.
  #
  # Dependencies: https://github.com/Maratyszcza/PeachPy
  #
  # python3 -m peachpy.x86_64 -mabi=goasm -S -o chacha20_amd64.s chacha20_amd64.py
  #
  
  from peachpy import *
  from peachpy.x86_64 import *
  
  x = Argument(ptr(uint32_t))
  inp = Argument(ptr(const_uint8_t))
  outp = Argument(ptr(uint8_t))
  nrBlocks = Argument(ptr(size_t))
  
  #
  # SSE2 helper functions.  A temporary register is explicitly passed in because
  # the main fast loop uses every single register (and even spills) so manual
  # control is needed.
  #
  # This used to also have a DQROUNDS helper that did 2 rounds of ChaCha like
  # in the C code, but the C code has the luxury of an optimizer reordering
  # everything, while this does not.
  #
  
  def ROTW16_sse2(tmp, d):
      MOVDQA(tmp, d)
      PSLLD(tmp, 16)
      PSRLD(d, 16)
      PXOR(d, tmp)
  
  def ROTW12_sse2(tmp, b):
      MOVDQA(tmp, b)
      PSLLD(tmp, 12)
      PSRLD(b, 20)
      PXOR(b, tmp)
  
  def ROTW8_sse2(tmp, d):
      MOVDQA(tmp, d)
      PSLLD(tmp, 8)
      PSRLD(d, 24)
      PXOR(d, tmp)
  
  def ROTW7_sse2(tmp, b):
      MOVDQA(tmp, b)
      PSLLD(tmp, 7)
      PSRLD(b, 25)
      PXOR(b, tmp)
  
  def WriteXor_sse2(tmp, inp, outp, d, v0, v1, v2, v3):
      MOVDQU(tmp, [inp+d])
      PXOR(tmp, v0)
      MOVDQU([outp+d], tmp)
      MOVDQU(tmp, [inp+d+16])
      PXOR(tmp, v1)
      MOVDQU([outp+d+16], tmp)
      MOVDQU(tmp, [inp+d+32])
      PXOR(tmp, v2)
      MOVDQU([outp+d+32], tmp)
      MOVDQU(tmp, [inp+d+48])
      PXOR(tmp, v3)
      MOVDQU([outp+d+48], tmp)
  
  # SSE2 ChaCha20 (aka vec128).  Does not handle partial blocks, and will
  # process 4/2/1 blocks at a time.
  with Function("blocksAmd64SSE2", (x, inp, outp, nrBlocks)):
      reg_x = GeneralPurposeRegister64()
      reg_inp = GeneralPurposeRegister64()
      reg_outp = GeneralPurposeRegister64()
      reg_blocks = GeneralPurposeRegister64()
      reg_sp_save = GeneralPurposeRegister64()
  
      LOAD.ARGUMENT(reg_x, x)
      LOAD.ARGUMENT(reg_inp, inp)
      LOAD.ARGUMENT(reg_outp, outp)
      LOAD.ARGUMENT(reg_blocks, nrBlocks)
  
      # Align the stack to a 32 byte boundary.
      MOV(reg_sp_save, registers.rsp)
      AND(registers.rsp, 0xffffffffffffffe0)
      SUB(registers.rsp, 0x20)
  
      # Build the counter increment vector on the stack, and allocate the scratch
      # space
      xmm_v0 = XMMRegister()
      PXOR(xmm_v0, xmm_v0)
      SUB(registers.rsp, 16+16)
      MOVDQA([registers.rsp], xmm_v0)
      reg_tmp = GeneralPurposeRegister32()
      MOV(reg_tmp, 0x00000001)
      MOV([registers.rsp], reg_tmp)
      mem_one = [registers.rsp]     # (Stack) Counter increment vector
      mem_tmp0 = [registers.rsp+16] # (Stack) Scratch space.
  
      mem_s0 = [reg_x]           # (Memory) Cipher state [0..3]
      mem_s1 = [reg_x+16]        # (Memory) Cipher state [4..7]
      mem_s2 = [reg_x+32]        # (Memory) Cipher state [8..11]
      mem_s3 = [reg_x+48]        # (Memory) Cipher state [12..15]
  
      # xmm_v0 allocated above...
      xmm_v1 = XMMRegister()
      xmm_v2 = XMMRegister()
      xmm_v3 = XMMRegister()
  
      xmm_v4 = XMMRegister()
      xmm_v5 = XMMRegister()
      xmm_v6 = XMMRegister()
      xmm_v7 = XMMRegister()
  
      xmm_v8 = XMMRegister()
      xmm_v9 = XMMRegister()
      xmm_v10 = XMMRegister()
      xmm_v11 = XMMRegister()
  
      xmm_v12 = XMMRegister()
      xmm_v13 = XMMRegister()
      xmm_v14 = XMMRegister()
      xmm_v15 = XMMRegister()
  
      xmm_tmp = xmm_v12
  
      #
      # 4 blocks at a time.
      #
  
      reg_rounds = GeneralPurposeRegister64()
  
      vector_loop4 = Loop()
      SUB(reg_blocks, 4)
      JB(vector_loop4.end)
      with vector_loop4:
          MOVDQU(xmm_v0, mem_s0)
          MOVDQU(xmm_v1, mem_s1)
          MOVDQU(xmm_v2, mem_s2)
          MOVDQU(xmm_v3, mem_s3)
  
          MOVDQA(xmm_v4, xmm_v0)
          MOVDQA(xmm_v5, xmm_v1)
          MOVDQA(xmm_v6, xmm_v2)
          MOVDQA(xmm_v7, xmm_v3)
          PADDQ(xmm_v7, mem_one)
  
          MOVDQA(xmm_v8, xmm_v0)
          MOVDQA(xmm_v9, xmm_v1)
          MOVDQA(xmm_v10, xmm_v2)
          MOVDQA(xmm_v11, xmm_v7)
          PADDQ(xmm_v11, mem_one)
  
          MOVDQA(xmm_v12, xmm_v0)
          MOVDQA(xmm_v13, xmm_v1)
          MOVDQA(xmm_v14, xmm_v2)
          MOVDQA(xmm_v15, xmm_v11)
          PADDQ(xmm_v15, mem_one)
  
          MOV(reg_rounds, 20)
          rounds_loop4 = Loop()
          with rounds_loop4:
              # a += b; d ^= a; d = ROTW16(d);
              PADDD(xmm_v0, xmm_v1)
              PADDD(xmm_v4, xmm_v5)
              PADDD(xmm_v8, xmm_v9)
              PADDD(xmm_v12, xmm_v13)
              PXOR(xmm_v3, xmm_v0)
              PXOR(xmm_v7, xmm_v4)
              PXOR(xmm_v11, xmm_v8)
              PXOR(xmm_v15, xmm_v12)
  
              MOVDQA(mem_tmp0, xmm_tmp) # Save
  
              ROTW16_sse2(xmm_tmp, xmm_v3)
              ROTW16_sse2(xmm_tmp, xmm_v7)
              ROTW16_sse2(xmm_tmp, xmm_v11)
              ROTW16_sse2(xmm_tmp, xmm_v15)
  
              # c += d; b ^= c; b = ROTW12(b);
              PADDD(xmm_v2, xmm_v3)
              PADDD(xmm_v6, xmm_v7)
              PADDD(xmm_v10, xmm_v11)
              PADDD(xmm_v14, xmm_v15)
              PXOR(xmm_v1, xmm_v2)
              PXOR(xmm_v5, xmm_v6)
              PXOR(xmm_v9, xmm_v10)
              PXOR(xmm_v13, xmm_v14)
              ROTW12_sse2(xmm_tmp, xmm_v1)
              ROTW12_sse2(xmm_tmp, xmm_v5)
              ROTW12_sse2(xmm_tmp, xmm_v9)
              ROTW12_sse2(xmm_tmp, xmm_v13)
  
              # a += b; d ^= a; d = ROTW8(d);
              MOVDQA(xmm_tmp, mem_tmp0) # Restore
  
              PADDD(xmm_v0, xmm_v1)
              PADDD(xmm_v4, xmm_v5)
              PADDD(xmm_v8, xmm_v9)
              PADDD(xmm_v12, xmm_v13)
              PXOR(xmm_v3, xmm_v0)
              PXOR(xmm_v7, xmm_v4)
              PXOR(xmm_v11, xmm_v8)
              PXOR(xmm_v15, xmm_v12)
  
              MOVDQA(mem_tmp0, xmm_tmp) # Save
  
              ROTW8_sse2(xmm_tmp, xmm_v3)
              ROTW8_sse2(xmm_tmp, xmm_v7)
              ROTW8_sse2(xmm_tmp, xmm_v11)
              ROTW8_sse2(xmm_tmp, xmm_v15)
  
              # c += d; b ^= c; b = ROTW7(b)
              PADDD(xmm_v2, xmm_v3)
              PADDD(xmm_v6, xmm_v7)
              PADDD(xmm_v10, xmm_v11)
              PADDD(xmm_v14, xmm_v15)
              PXOR(xmm_v1, xmm_v2)
              PXOR(xmm_v5, xmm_v6)
              PXOR(xmm_v9, xmm_v10)
              PXOR(xmm_v13, xmm_v14)
              ROTW7_sse2(xmm_tmp, xmm_v1)
              ROTW7_sse2(xmm_tmp, xmm_v5)
              ROTW7_sse2(xmm_tmp, xmm_v9)
              ROTW7_sse2(xmm_tmp, xmm_v13)
  
              # b = ROTV1(b); c = ROTV2(c);  d = ROTV3(d);
              PSHUFD(xmm_v1, xmm_v1, 0x39)
              PSHUFD(xmm_v5, xmm_v5, 0x39)
              PSHUFD(xmm_v9, xmm_v9, 0x39)
              PSHUFD(xmm_v13, xmm_v13, 0x39)
              PSHUFD(xmm_v2, xmm_v2, 0x4e)
              PSHUFD(xmm_v6, xmm_v6, 0x4e)
              PSHUFD(xmm_v10, xmm_v10, 0x4e)
              PSHUFD(xmm_v14, xmm_v14, 0x4e)
              PSHUFD(xmm_v3, xmm_v3, 0x93)
              PSHUFD(xmm_v7, xmm_v7, 0x93)
              PSHUFD(xmm_v11, xmm_v11, 0x93)
              PSHUFD(xmm_v15, xmm_v15, 0x93)
  
              MOVDQA(xmm_tmp, mem_tmp0) # Restore
  
              # a += b; d ^= a; d = ROTW16(d);
              PADDD(xmm_v0, xmm_v1)
              PADDD(xmm_v4, xmm_v5)
              PADDD(xmm_v8, xmm_v9)
              PADDD(xmm_v12, xmm_v13)
              PXOR(xmm_v3, xmm_v0)
              PXOR(xmm_v7, xmm_v4)
              PXOR(xmm_v11, xmm_v8)
              PXOR(xmm_v15, xmm_v12)
  
              MOVDQA(mem_tmp0, xmm_tmp) # Save
  
              ROTW16_sse2(xmm_tmp, xmm_v3)
              ROTW16_sse2(xmm_tmp, xmm_v7)
              ROTW16_sse2(xmm_tmp, xmm_v11)
              ROTW16_sse2(xmm_tmp, xmm_v15)
  
              # c += d; b ^= c; b = ROTW12(b);
              PADDD(xmm_v2, xmm_v3)
              PADDD(xmm_v6, xmm_v7)
              PADDD(xmm_v10, xmm_v11)
              PADDD(xmm_v14, xmm_v15)
              PXOR(xmm_v1, xmm_v2)
              PXOR(xmm_v5, xmm_v6)
              PXOR(xmm_v9, xmm_v10)
              PXOR(xmm_v13, xmm_v14)
              ROTW12_sse2(xmm_tmp, xmm_v1)
              ROTW12_sse2(xmm_tmp, xmm_v5)
              ROTW12_sse2(xmm_tmp, xmm_v9)
              ROTW12_sse2(xmm_tmp, xmm_v13)
  
              # a += b; d ^= a; d = ROTW8(d);
              MOVDQA(xmm_tmp, mem_tmp0) # Restore
  
              PADDD(xmm_v0, xmm_v1)
              PADDD(xmm_v4, xmm_v5)
              PADDD(xmm_v8, xmm_v9)
              PADDD(xmm_v12, xmm_v13)
              PXOR(xmm_v3, xmm_v0)
              PXOR(xmm_v7, xmm_v4)
              PXOR(xmm_v11, xmm_v8)
              PXOR(xmm_v15, xmm_v12)
  
              MOVDQA(mem_tmp0, xmm_tmp) # Save
  
              ROTW8_sse2(xmm_tmp, xmm_v3)
              ROTW8_sse2(xmm_tmp, xmm_v7)
              ROTW8_sse2(xmm_tmp, xmm_v11)
              ROTW8_sse2(xmm_tmp, xmm_v15)
  
              # c += d; b ^= c; b = ROTW7(b)
              PADDD(xmm_v2, xmm_v3)
              PADDD(xmm_v6, xmm_v7)
              PADDD(xmm_v10, xmm_v11)
              PADDD(xmm_v14, xmm_v15)
              PXOR(xmm_v1, xmm_v2)
              PXOR(xmm_v5, xmm_v6)
              PXOR(xmm_v9, xmm_v10)
              PXOR(xmm_v13, xmm_v14)
              ROTW7_sse2(xmm_tmp, xmm_v1)
              ROTW7_sse2(xmm_tmp, xmm_v5)
              ROTW7_sse2(xmm_tmp, xmm_v9)
              ROTW7_sse2(xmm_tmp, xmm_v13)
  
              # b = ROTV1(b); c = ROTV2(c);  d = ROTV3(d);
              PSHUFD(xmm_v1, xmm_v1, 0x93)
              PSHUFD(xmm_v5, xmm_v5, 0x93)
              PSHUFD(xmm_v9, xmm_v9, 0x93)
              PSHUFD(xmm_v13, xmm_v13, 0x93)
              PSHUFD(xmm_v2, xmm_v2, 0x4e)
              PSHUFD(xmm_v6, xmm_v6, 0x4e)
              PSHUFD(xmm_v10, xmm_v10, 0x4e)
              PSHUFD(xmm_v14, xmm_v14, 0x4e)
              PSHUFD(xmm_v3, xmm_v3, 0x39)
              PSHUFD(xmm_v7, xmm_v7, 0x39)
              PSHUFD(xmm_v11, xmm_v11, 0x39)
              PSHUFD(xmm_v15, xmm_v15, 0x39)
  
              MOVDQA(xmm_tmp, mem_tmp0) # Restore
  
              SUB(reg_rounds, 2)
              JNZ(rounds_loop4.begin)
  
          MOVDQA(mem_tmp0, xmm_tmp)
  
          PADDD(xmm_v0, mem_s0)
          PADDD(xmm_v1, mem_s1)
          PADDD(xmm_v2, mem_s2)
          PADDD(xmm_v3, mem_s3)
          WriteXor_sse2(xmm_tmp, reg_inp, reg_outp, 0, xmm_v0, xmm_v1, xmm_v2, xmm_v3)
          MOVDQU(xmm_v3, mem_s3)
          PADDQ(xmm_v3, mem_one)
  
          PADDD(xmm_v4, mem_s0)
          PADDD(xmm_v5, mem_s1)
          PADDD(xmm_v6, mem_s2)
          PADDD(xmm_v7, xmm_v3)
          WriteXor_sse2(xmm_tmp, reg_inp, reg_outp, 64, xmm_v4, xmm_v5, xmm_v6, xmm_v7)
          PADDQ(xmm_v3, mem_one)
  
          PADDD(xmm_v8, mem_s0)
          PADDD(xmm_v9, mem_s1)
          PADDD(xmm_v10, mem_s2)
          PADDD(xmm_v11, xmm_v3)
          WriteXor_sse2(xmm_tmp, reg_inp, reg_outp, 128, xmm_v8, xmm_v9, xmm_v10, xmm_v11)
          PADDQ(xmm_v3, mem_one)
  
          MOVDQA(xmm_tmp, mem_tmp0)
  
          PADDD(xmm_v12, mem_s0)
          PADDD(xmm_v13, mem_s1)
          PADDD(xmm_v14, mem_s2)
          PADDD(xmm_v15, xmm_v3)
          WriteXor_sse2(xmm_v0, reg_inp, reg_outp, 192, xmm_v12, xmm_v13, xmm_v14, xmm_v15)
          PADDQ(xmm_v3, mem_one)
  
          MOVDQU(mem_s3, xmm_v3)
  
          ADD(reg_inp, 4 * 64)
          ADD(reg_outp, 4 * 64)
  
          SUB(reg_blocks, 4)
          JAE(vector_loop4.begin)
  
      ADD(reg_blocks, 4)
      out = Label()
      JZ(out)
  
      # Past this point, we no longer need to use every single register to hold
      # the in progress state.
  
      xmm_s0 = xmm_v8
      xmm_s1 = xmm_v9
      xmm_s2 = xmm_v10
      xmm_s3 = xmm_v11
      xmm_one = xmm_v13
      MOVDQU(xmm_s0, mem_s0)
      MOVDQU(xmm_s1, mem_s1)
      MOVDQU(xmm_s2, mem_s2)
      MOVDQU(xmm_s3, mem_s3)
      MOVDQA(xmm_one, mem_one)
  
      #
      # 2 blocks at a time.
      #
  
      process_1_block = Label()
      SUB(reg_blocks, 2)
      JB(process_1_block) # < 2 blocks remaining.
  
      MOVDQA(xmm_v0, xmm_s0)
      MOVDQA(xmm_v1, xmm_s1)
      MOVDQA(xmm_v2, xmm_s2)
      MOVDQA(xmm_v3, xmm_s3)
  
      MOVDQA(xmm_v4, xmm_v0)
      MOVDQA(xmm_v5, xmm_v1)
      MOVDQA(xmm_v6, xmm_v2)
      MOVDQA(xmm_v7, xmm_v3)
      PADDQ(xmm_v7, xmm_one)
  
      MOV(reg_rounds, 20)
      rounds_loop2 = Loop()
      with rounds_loop2:
          # a += b; d ^= a; d = ROTW16(d);
          PADDD(xmm_v0, xmm_v1)
          PADDD(xmm_v4, xmm_v5)
          PXOR(xmm_v3, xmm_v0)
          PXOR(xmm_v7, xmm_v4)
          ROTW16_sse2(xmm_tmp, xmm_v3)
          ROTW16_sse2(xmm_tmp, xmm_v7)
  
          # c += d; b ^= c; b = ROTW12(b);
          PADDD(xmm_v2, xmm_v3)
          PADDD(xmm_v6, xmm_v7)
          PXOR(xmm_v1, xmm_v2)
          PXOR(xmm_v5, xmm_v6)
          ROTW12_sse2(xmm_tmp, xmm_v1)
          ROTW12_sse2(xmm_tmp, xmm_v5)
  
          # a += b; d ^= a; d = ROTW8(d);
          PADDD(xmm_v0, xmm_v1)
          PADDD(xmm_v4, xmm_v5)
          PXOR(xmm_v3, xmm_v0)
          PXOR(xmm_v7, xmm_v4)
          ROTW8_sse2(xmm_tmp, xmm_v3)
          ROTW8_sse2(xmm_tmp, xmm_v7)
  
          # c += d; b ^= c; b = ROTW7(b)
          PADDD(xmm_v2, xmm_v3)
          PADDD(xmm_v6, xmm_v7)
          PXOR(xmm_v1, xmm_v2)
          PXOR(xmm_v5, xmm_v6)
          ROTW7_sse2(xmm_tmp, xmm_v1)
          ROTW7_sse2(xmm_tmp, xmm_v5)
  
          # b = ROTV1(b); c = ROTV2(c);  d = ROTV3(d);
          PSHUFD(xmm_v1, xmm_v1, 0x39)
          PSHUFD(xmm_v5, xmm_v5, 0x39)
          PSHUFD(xmm_v2, xmm_v2, 0x4e)
          PSHUFD(xmm_v6, xmm_v6, 0x4e)
          PSHUFD(xmm_v3, xmm_v3, 0x93)
          PSHUFD(xmm_v7, xmm_v7, 0x93)
  
          # a += b; d ^= a; d = ROTW16(d);
          PADDD(xmm_v0, xmm_v1)
          PADDD(xmm_v4, xmm_v5)
          PXOR(xmm_v3, xmm_v0)
          PXOR(xmm_v7, xmm_v4)
          ROTW16_sse2(xmm_tmp, xmm_v3)
          ROTW16_sse2(xmm_tmp, xmm_v7)
  
          # c += d; b ^= c; b = ROTW12(b);
          PADDD(xmm_v2, xmm_v3)
          PADDD(xmm_v6, xmm_v7)
          PXOR(xmm_v1, xmm_v2)
          PXOR(xmm_v5, xmm_v6)
          ROTW12_sse2(xmm_tmp, xmm_v1)
          ROTW12_sse2(xmm_tmp, xmm_v5)
  
          # a += b; d ^= a; d = ROTW8(d);
          PADDD(xmm_v0, xmm_v1)
          PADDD(xmm_v4, xmm_v5)
          PXOR(xmm_v3, xmm_v0)
          PXOR(xmm_v7, xmm_v4)
          ROTW8_sse2(xmm_tmp, xmm_v3)
          ROTW8_sse2(xmm_tmp, xmm_v7)
  
          # c += d; b ^= c; b = ROTW7(b)
          PADDD(xmm_v2, xmm_v3)
          PADDD(xmm_v6, xmm_v7)
          PXOR(xmm_v1, xmm_v2)
          PXOR(xmm_v5, xmm_v6)
          ROTW7_sse2(xmm_tmp, xmm_v1)
          ROTW7_sse2(xmm_tmp, xmm_v5)
  
          # b = ROTV1(b); c = ROTV2(c);  d = ROTV3(d);
          PSHUFD(xmm_v1, xmm_v1, 0x93)
          PSHUFD(xmm_v5, xmm_v5, 0x93)
          PSHUFD(xmm_v2, xmm_v2, 0x4e)
          PSHUFD(xmm_v6, xmm_v6, 0x4e)
          PSHUFD(xmm_v3, xmm_v3, 0x39)
          PSHUFD(xmm_v7, xmm_v7, 0x39)
  
          SUB(reg_rounds, 2)
          JNZ(rounds_loop2.begin)
  
      PADDD(xmm_v0, xmm_s0)
      PADDD(xmm_v1, xmm_s1)
      PADDD(xmm_v2, xmm_s2)
      PADDD(xmm_v3, xmm_s3)
      WriteXor_sse2(xmm_tmp, reg_inp, reg_outp, 0, xmm_v0, xmm_v1, xmm_v2, xmm_v3)
      PADDQ(xmm_s3, xmm_one)
  
      PADDD(xmm_v4, xmm_s0)
      PADDD(xmm_v5, xmm_s1)
      PADDD(xmm_v6, xmm_s2)
      PADDD(xmm_v7, xmm_s3)
      WriteXor_sse2(xmm_tmp, reg_inp, reg_outp, 64, xmm_v4, xmm_v5, xmm_v6, xmm_v7)
      PADDQ(xmm_s3, xmm_one)
  
      ADD(reg_inp, 2 * 64)
      ADD(reg_outp, 2 * 64)
      SUB(reg_blocks, 2)
  
      LABEL(process_1_block)
      ADD(reg_blocks, 2)
      out_serial = Label()
      JZ(out_serial)
  
      #
      # 1 block at a time.  Only executed once, because if there was > 1,
      # the parallel code would have processed it already.
      #
  
      MOVDQA(xmm_v0, xmm_s0)
      MOVDQA(xmm_v1, xmm_s1)
      MOVDQA(xmm_v2, xmm_s2)
      MOVDQA(xmm_v3, xmm_s3)
  
      MOV(reg_rounds, 20)
      rounds_loop1 = Loop()
      with rounds_loop1:
          # a += b; d ^= a; d = ROTW16(d);
          PADDD(xmm_v0, xmm_v1)
          PXOR(xmm_v3, xmm_v0)
          ROTW16_sse2(xmm_tmp, xmm_v3)
  
          # c += d; b ^= c; b = ROTW12(b);
          PADDD(xmm_v2, xmm_v3)
          PXOR(xmm_v1, xmm_v2)
          ROTW12_sse2(xmm_tmp, xmm_v1)
  
          # a += b; d ^= a; d = ROTW8(d);
          PADDD(xmm_v0, xmm_v1)
          PXOR(xmm_v3, xmm_v0)
          ROTW8_sse2(xmm_tmp, xmm_v3)
  
          # c += d; b ^= c; b = ROTW7(b)
          PADDD(xmm_v2, xmm_v3)
          PXOR(xmm_v1, xmm_v2)
          ROTW7_sse2(xmm_tmp, xmm_v1)
  
          # b = ROTV1(b); c = ROTV2(c);  d = ROTV3(d);
          PSHUFD(xmm_v1, xmm_v1, 0x39)
          PSHUFD(xmm_v2, xmm_v2, 0x4e)
          PSHUFD(xmm_v3, xmm_v3, 0x93)
  
          # a += b; d ^= a; d = ROTW16(d);
          PADDD(xmm_v0, xmm_v1)
          PXOR(xmm_v3, xmm_v0)
          ROTW16_sse2(xmm_tmp, xmm_v3)
  
          # c += d; b ^= c; b = ROTW12(b);
          PADDD(xmm_v2, xmm_v3)
          PXOR(xmm_v1, xmm_v2)
          ROTW12_sse2(xmm_tmp, xmm_v1)
  
          # a += b; d ^= a; d = ROTW8(d);
          PADDD(xmm_v0, xmm_v1)
          PXOR(xmm_v3, xmm_v0)
          ROTW8_sse2(xmm_tmp, xmm_v3)
  
          # c += d; b ^= c; b = ROTW7(b)
          PADDD(xmm_v2, xmm_v3)
          PXOR(xmm_v1, xmm_v2)
          ROTW7_sse2(xmm_tmp, xmm_v1)
  
          # b = ROTV1(b); c = ROTV2(c);  d = ROTV3(d);
          PSHUFD(xmm_v1, xmm_v1, 0x93)
          PSHUFD(xmm_v2, xmm_v2, 0x4e)
          PSHUFD(xmm_v3, xmm_v3, 0x39)
  
          SUB(reg_rounds, 2)
          JNZ(rounds_loop1.begin)
  
      PADDD(xmm_v0, xmm_s0)
      PADDD(xmm_v1, xmm_s1)
      PADDD(xmm_v2, xmm_s2)
      PADDD(xmm_v3, xmm_s3)
      WriteXor_sse2(xmm_tmp, reg_inp, reg_outp, 0, xmm_v0, xmm_v1, xmm_v2, xmm_v3)
      PADDQ(xmm_s3, xmm_one)
  
      LABEL(out_serial)
  
      # Write back the updated counter.  Stoping at 2^70 bytes is the user's
      # problem, not mine.  (Skipped if there's exactly a multiple of 4 blocks
      # because the counter is incremented in memory while looping.)
      MOVDQU(mem_s3, xmm_s3)
  
      LABEL(out)
  
      # Paranoia, cleanse the scratch space.
      PXOR(xmm_v0, xmm_v0)
      MOVDQA(mem_tmp0, xmm_v0)
  
      # Remove our stack allocation.
      MOV(registers.rsp, reg_sp_save)
  
      RETURN()
  
  #
  # AVX2 helpers.  Like the SSE2 equivalents, the scratch register is explicit,
  # and more helpers are used to increase readability for destructive operations.
  #
  # XXX/Performance: ROTW16_avx2/ROTW8_avx2 both can use VPSHUFFB.
  #
  
  def ADD_avx2(dst, src):
      VPADDD(dst, dst, src)
  
  def XOR_avx2(dst, src):
      VPXOR(dst, dst, src)
  
  def ROTW16_avx2(tmp, d):
      VPSLLD(tmp, d, 16)
      VPSRLD(d, d, 16)
      XOR_avx2(d, tmp)
  
  def ROTW12_avx2(tmp, b):
      VPSLLD(tmp, b, 12)
      VPSRLD(b, b, 20)
      XOR_avx2(b, tmp)
  
  def ROTW8_avx2(tmp, d):
      VPSLLD(tmp, d, 8)
      VPSRLD(d, d, 24)
      XOR_avx2(d, tmp)
  
  def ROTW7_avx2(tmp, b):
      VPSLLD(tmp, b, 7)
      VPSRLD(b, b, 25)
      XOR_avx2(b, tmp)
  
  def WriteXor_avx2(tmp, inp, outp, d, v0, v1, v2, v3):
      # XOR_WRITE(out+ 0, in+ 0, _mm256_permute2x128_si256(v0,v1,0x20));
      VPERM2I128(tmp, v0, v1, 0x20)
      VPXOR(tmp, tmp, [inp+d])
      VMOVDQU([outp+d], tmp)
  
      # XOR_WRITE(out+32, in+32, _mm256_permute2x128_si256(v2,v3,0x20));
      VPERM2I128(tmp, v2, v3, 0x20)
      VPXOR(tmp, tmp, [inp+d+32])
      VMOVDQU([outp+d+32], tmp)
  
      # XOR_WRITE(out+64, in+64, _mm256_permute2x128_si256(v0,v1,0x31));
      VPERM2I128(tmp, v0, v1, 0x31)
      VPXOR(tmp, tmp, [inp+d+64])
      VMOVDQU([outp+d+64], tmp)
  
      # XOR_WRITE(out+96, in+96, _mm256_permute2x128_si256(v2,v3,0x31));
      VPERM2I128(tmp, v2, v3, 0x31)
      VPXOR(tmp, tmp, [inp+d+96])
      VMOVDQU([outp+d+96], tmp)
  
  # AVX2 ChaCha20 (aka avx2).  Does not handle partial blocks, will process
  # 8/4/2 blocks at a time.
  with Function("blocksAmd64AVX2", (x, inp, outp, nrBlocks), target=uarch.broadwell):
      reg_x = GeneralPurposeRegister64()
      reg_inp = GeneralPurposeRegister64()
      reg_outp = GeneralPurposeRegister64()
      reg_blocks = GeneralPurposeRegister64()
      reg_sp_save = GeneralPurposeRegister64()
  
      LOAD.ARGUMENT(reg_x, x)
      LOAD.ARGUMENT(reg_inp, inp)
      LOAD.ARGUMENT(reg_outp, outp)
      LOAD.ARGUMENT(reg_blocks, nrBlocks)
  
      # Align the stack to a 32 byte boundary.
      MOV(reg_sp_save, registers.rsp)
      AND(registers.rsp, 0xffffffffffffffe0)
      SUB(registers.rsp, 0x20)
  
      x_s0 = [reg_x]           # (Memory) Cipher state [0..3]
      x_s1 = [reg_x+16]        # (Memory) Cipher state [4..7]
      x_s2 = [reg_x+32]        # (Memory) Cipher state [8..11]
      x_s3 = [reg_x+48]        # (Memory) Cipher state [12..15]
  
      ymm_v0 = YMMRegister()
      ymm_v1 = YMMRegister()
      ymm_v2 = YMMRegister()
      ymm_v3 = YMMRegister()
  
      ymm_v4 = YMMRegister()
      ymm_v5 = YMMRegister()
      ymm_v6 = YMMRegister()
      ymm_v7 = YMMRegister()
  
      ymm_v8 = YMMRegister()
      ymm_v9 = YMMRegister()
      ymm_v10 = YMMRegister()
      ymm_v11 = YMMRegister()
  
      ymm_v12 = YMMRegister()
      ymm_v13 = YMMRegister()
      ymm_v14 = YMMRegister()
      ymm_v15 = YMMRegister()
  
      ymm_tmp0 = ymm_v12
  
      # Allocate the neccecary stack space for the counter vector and two ymm
      # registers that we will spill.
      SUB(registers.rsp, 96)
      mem_tmp0 = [registers.rsp+64]  # (Stack) Scratch space.
      mem_s3 = [registers.rsp+32]    # (Stack) Working copy of s3. (8x)
      mem_inc = [registers.rsp]      # (Stack) Counter increment vector.
  
      # Increment the counter for one side of the state vector.
      VPXOR(ymm_tmp0, ymm_tmp0, ymm_tmp0)
      VMOVDQU(mem_inc, ymm_tmp0)
      reg_tmp = GeneralPurposeRegister32()
      MOV(reg_tmp, 0x00000001)
      MOV([registers.rsp+16], reg_tmp)
      VBROADCASTI128(ymm_v3, x_s3)
      VPADDQ(ymm_v3, ymm_v3, [registers.rsp])
      VMOVDQA(mem_s3, ymm_v3)
  
      # As we process 2xN blocks at a time, so the counter increment for both
      # sides of the state vector is 2.
      MOV(reg_tmp, 0x00000002)
      MOV([registers.rsp], reg_tmp)
      MOV([registers.rsp+16], reg_tmp)
  
      out_write_even = Label()
      out_write_odd = Label()
  
      #
      # 8 blocks at a time.  Ted Krovetz's avx2 code does not do this, but it's
      # a decent gain despite all the pain...
      #
  
      reg_rounds = GeneralPurposeRegister64()
  
      vector_loop8 = Loop()
      SUB(reg_blocks, 8)
      JB(vector_loop8.end)
      with vector_loop8:
          VBROADCASTI128(ymm_v0, x_s0)
          VBROADCASTI128(ymm_v1, x_s1)
          VBROADCASTI128(ymm_v2, x_s2)
          VMOVDQA(ymm_v3, mem_s3)
  
          VMOVDQA(ymm_v4, ymm_v0)
          VMOVDQA(ymm_v5, ymm_v1)
          VMOVDQA(ymm_v6, ymm_v2)
          VPADDQ(ymm_v7, ymm_v3, mem_inc)
  
          VMOVDQA(ymm_v8, ymm_v0)
          VMOVDQA(ymm_v9, ymm_v1)
          VMOVDQA(ymm_v10, ymm_v2)
          VPADDQ(ymm_v11, ymm_v7, mem_inc)
  
          VMOVDQA(ymm_v12, ymm_v0)
          VMOVDQA(ymm_v13, ymm_v1)
          VMOVDQA(ymm_v14, ymm_v2)
          VPADDQ(ymm_v15, ymm_v11, mem_inc)
  
          MOV(reg_rounds, 20)
          rounds_loop8 = Loop()
          with rounds_loop8:
              # a += b; d ^= a; d = ROTW16(d);
              ADD_avx2(ymm_v0, ymm_v1)
              ADD_avx2(ymm_v4, ymm_v5)
              ADD_avx2(ymm_v8, ymm_v9)
              ADD_avx2(ymm_v12, ymm_v13)
              XOR_avx2(ymm_v3, ymm_v0)
              XOR_avx2(ymm_v7, ymm_v4)
              XOR_avx2(ymm_v11, ymm_v8)
              XOR_avx2(ymm_v15, ymm_v12)
  
              VMOVDQA(mem_tmp0, ymm_tmp0) # Save
  
              ROTW16_avx2(ymm_tmp0, ymm_v3)
              ROTW16_avx2(ymm_tmp0, ymm_v7)
              ROTW16_avx2(ymm_tmp0, ymm_v11)
              ROTW16_avx2(ymm_tmp0, ymm_v15)
  
              # c += d; b ^= c; b = ROTW12(b);
              ADD_avx2(ymm_v2, ymm_v3)
              ADD_avx2(ymm_v6, ymm_v7)
              ADD_avx2(ymm_v10, ymm_v11)
              ADD_avx2(ymm_v14, ymm_v15)
              XOR_avx2(ymm_v1, ymm_v2)
              XOR_avx2(ymm_v5, ymm_v6)
              XOR_avx2(ymm_v9, ymm_v10)
              XOR_avx2(ymm_v13, ymm_v14)
              ROTW12_avx2(ymm_tmp0, ymm_v1)
              ROTW12_avx2(ymm_tmp0, ymm_v5)
              ROTW12_avx2(ymm_tmp0, ymm_v9)
              ROTW12_avx2(ymm_tmp0, ymm_v13)
  
              # a += b; d ^= a; d = ROTW8(d);
              VMOVDQA(ymm_tmp0, mem_tmp0) # Restore
  
              ADD_avx2(ymm_v0, ymm_v1)
              ADD_avx2(ymm_v4, ymm_v5)
              ADD_avx2(ymm_v8, ymm_v9)
              ADD_avx2(ymm_v12, ymm_v13)
              XOR_avx2(ymm_v3, ymm_v0)
              XOR_avx2(ymm_v7, ymm_v4)
              XOR_avx2(ymm_v11, ymm_v8)
              XOR_avx2(ymm_v15, ymm_v12)
  
              VMOVDQA(mem_tmp0, ymm_tmp0) # Save
  
              ROTW8_avx2(ymm_tmp0, ymm_v3)
              ROTW8_avx2(ymm_tmp0, ymm_v7)
              ROTW8_avx2(ymm_tmp0, ymm_v11)
              ROTW8_avx2(ymm_tmp0, ymm_v15)
  
              # c += d; b ^= c; b = ROTW7(b)
              ADD_avx2(ymm_v2, ymm_v3)
              ADD_avx2(ymm_v6, ymm_v7)
              ADD_avx2(ymm_v10, ymm_v11)
              ADD_avx2(ymm_v14, ymm_v15)
              XOR_avx2(ymm_v1, ymm_v2)
              XOR_avx2(ymm_v5, ymm_v6)
              XOR_avx2(ymm_v9, ymm_v10)
              XOR_avx2(ymm_v13, ymm_v14)
              ROTW7_avx2(ymm_tmp0, ymm_v1)
              ROTW7_avx2(ymm_tmp0, ymm_v5)
              ROTW7_avx2(ymm_tmp0, ymm_v9)
              ROTW7_avx2(ymm_tmp0, ymm_v13)
  
              # b = ROTV1(b); c = ROTV2(c);  d = ROTV3(d);
              VPSHUFD(ymm_v1, ymm_v1, 0x39)
              VPSHUFD(ymm_v5, ymm_v5, 0x39)
              VPSHUFD(ymm_v9, ymm_v9, 0x39)
              VPSHUFD(ymm_v13, ymm_v13, 0x39)
              VPSHUFD(ymm_v2, ymm_v2, 0x4e)
              VPSHUFD(ymm_v6, ymm_v6, 0x4e)
              VPSHUFD(ymm_v10, ymm_v10, 0x4e)
              VPSHUFD(ymm_v14, ymm_v14, 0x4e)
              VPSHUFD(ymm_v3, ymm_v3, 0x93)
              VPSHUFD(ymm_v7, ymm_v7, 0x93)
              VPSHUFD(ymm_v11, ymm_v11, 0x93)
              VPSHUFD(ymm_v15, ymm_v15, 0x93)
  
              # a += b; d ^= a; d = ROTW16(d);
              VMOVDQA(ymm_tmp0, mem_tmp0) # Restore
  
              ADD_avx2(ymm_v0, ymm_v1)
              ADD_avx2(ymm_v4, ymm_v5)
              ADD_avx2(ymm_v8, ymm_v9)
              ADD_avx2(ymm_v12, ymm_v13)
              XOR_avx2(ymm_v3, ymm_v0)
              XOR_avx2(ymm_v7, ymm_v4)
              XOR_avx2(ymm_v11, ymm_v8)
              XOR_avx2(ymm_v15, ymm_v12)
  
              VMOVDQA(mem_tmp0, ymm_tmp0) # Save
  
              ROTW16_avx2(ymm_tmp0, ymm_v3)
              ROTW16_avx2(ymm_tmp0, ymm_v7)
              ROTW16_avx2(ymm_tmp0, ymm_v11)
              ROTW16_avx2(ymm_tmp0, ymm_v15)
  
              # c += d; b ^= c; b = ROTW12(b);
              ADD_avx2(ymm_v2, ymm_v3)
              ADD_avx2(ymm_v6, ymm_v7)
              ADD_avx2(ymm_v10, ymm_v11)
              ADD_avx2(ymm_v14, ymm_v15)
              XOR_avx2(ymm_v1, ymm_v2)
              XOR_avx2(ymm_v5, ymm_v6)
              XOR_avx2(ymm_v9, ymm_v10)
              XOR_avx2(ymm_v13, ymm_v14)
              ROTW12_avx2(ymm_tmp0, ymm_v1)
              ROTW12_avx2(ymm_tmp0, ymm_v5)
              ROTW12_avx2(ymm_tmp0, ymm_v9)
              ROTW12_avx2(ymm_tmp0, ymm_v13)
  
              # a += b; d ^= a; d = ROTW8(d);
              VMOVDQA(ymm_tmp0, mem_tmp0) # Restore
  
              ADD_avx2(ymm_v0, ymm_v1)
              ADD_avx2(ymm_v4, ymm_v5)
              ADD_avx2(ymm_v8, ymm_v9)
              ADD_avx2(ymm_v12, ymm_v13)
              XOR_avx2(ymm_v3, ymm_v0)
              XOR_avx2(ymm_v7, ymm_v4)
              XOR_avx2(ymm_v11, ymm_v8)
              XOR_avx2(ymm_v15, ymm_v12)
  
              VMOVDQA(mem_tmp0, ymm_tmp0) # Save
  
              ROTW8_avx2(ymm_tmp0, ymm_v3)
              ROTW8_avx2(ymm_tmp0, ymm_v7)
              ROTW8_avx2(ymm_tmp0, ymm_v11)
              ROTW8_avx2(ymm_tmp0, ymm_v15)
  
              # c += d; b ^= c; b = ROTW7(b)
              ADD_avx2(ymm_v2, ymm_v3)
              ADD_avx2(ymm_v6, ymm_v7)
              ADD_avx2(ymm_v10, ymm_v11)
              ADD_avx2(ymm_v14, ymm_v15)
              XOR_avx2(ymm_v1, ymm_v2)
              XOR_avx2(ymm_v5, ymm_v6)
              XOR_avx2(ymm_v9, ymm_v10)
              XOR_avx2(ymm_v13, ymm_v14)
              ROTW7_avx2(ymm_tmp0, ymm_v1)
              ROTW7_avx2(ymm_tmp0, ymm_v5)
              ROTW7_avx2(ymm_tmp0, ymm_v9)
              ROTW7_avx2(ymm_tmp0, ymm_v13)
  
              # b = ROTV1(b); c = ROTV2(c);  d = ROTV3(d);
              VPSHUFD(ymm_v1, ymm_v1, 0x93)
              VPSHUFD(ymm_v5, ymm_v5, 0x93)
              VPSHUFD(ymm_v9, ymm_v9, 0x93)
              VPSHUFD(ymm_v13, ymm_v13, 0x93)
              VPSHUFD(ymm_v2, ymm_v2, 0x4e)
              VPSHUFD(ymm_v6, ymm_v6, 0x4e)
              VPSHUFD(ymm_v10, ymm_v10, 0x4e)
              VPSHUFD(ymm_v14, ymm_v14, 0x4e)
              VPSHUFD(ymm_v3, ymm_v3, 0x39)
              VPSHUFD(ymm_v7, ymm_v7, 0x39)
              VPSHUFD(ymm_v11, ymm_v11, 0x39)
              VPSHUFD(ymm_v15, ymm_v15, 0x39)
  
              VMOVDQA(ymm_tmp0, mem_tmp0) # Restore
  
              SUB(reg_rounds, 2)
              JNZ(rounds_loop8.begin)
  
          # ymm_v12 is in mem_tmp0 and is current....
  
          # XXX: I assume VBROADCASTI128 is about as fast as VMOVDQA....
          VBROADCASTI128(ymm_tmp0, x_s0)
          ADD_avx2(ymm_v0, ymm_tmp0)
          ADD_avx2(ymm_v4, ymm_tmp0)
          ADD_avx2(ymm_v8, ymm_tmp0)
          ADD_avx2(ymm_tmp0, mem_tmp0)
          VMOVDQA(mem_tmp0, ymm_tmp0)
  
          VBROADCASTI128(ymm_tmp0, x_s1)
          ADD_avx2(ymm_v1, ymm_tmp0)
          ADD_avx2(ymm_v5, ymm_tmp0)
          ADD_avx2(ymm_v9, ymm_tmp0)
          ADD_avx2(ymm_v13, ymm_tmp0)
  
          VBROADCASTI128(ymm_tmp0, x_s2)
          ADD_avx2(ymm_v2, ymm_tmp0)
          ADD_avx2(ymm_v6, ymm_tmp0)
          ADD_avx2(ymm_v10, ymm_tmp0)
          ADD_avx2(ymm_v14, ymm_tmp0)
  
          ADD_avx2(ymm_v3, mem_s3)
          WriteXor_avx2(ymm_tmp0, reg_inp, reg_outp, 0, ymm_v0, ymm_v1, ymm_v2, ymm_v3)
          VMOVDQA(ymm_v3, mem_s3)
          ADD_avx2(ymm_v3, mem_inc)
  
          ADD_avx2(ymm_v7, ymm_v3)
          WriteXor_avx2(ymm_tmp0, reg_inp, reg_outp, 128, ymm_v4, ymm_v5, ymm_v6, ymm_v7)
          ADD_avx2(ymm_v3, mem_inc)
  
          ADD_avx2(ymm_v11, ymm_v3)
          WriteXor_avx2(ymm_tmp0, reg_inp, reg_outp, 256, ymm_v8, ymm_v9, ymm_v10, ymm_v11)
          ADD_avx2(ymm_v3, mem_inc)
  
          VMOVDQA(ymm_v12, mem_tmp0)
          ADD_avx2(ymm_v15, ymm_v3)
          WriteXor_avx2(ymm_v0, reg_inp, reg_outp, 384, ymm_v12, ymm_v13, ymm_v14, ymm_v15)
          ADD_avx2(ymm_v3, mem_inc)
  
          VMOVDQA(mem_s3, ymm_v3)
  
          ADD(reg_inp, 8 * 64)
          ADD(reg_outp, 8 * 64)
  
          SUB(reg_blocks, 8)
          JAE(vector_loop8.begin)
  
      # ymm_v3 contains a current copy of mem_s3 either from when it was built,
      # or because the loop updates it.  Copy this before we mess with the block
      # counter in case we need to write it back and return.
      ymm_s3 = ymm_v11
      VMOVDQA(ymm_s3, ymm_v3)
  
      ADD(reg_blocks, 8)
      JZ(out_write_even)
  
      # We now actually can do everything in registers.
      ymm_s0 = ymm_v8
      VBROADCASTI128(ymm_s0, x_s0)
      ymm_s1 = ymm_v9
      VBROADCASTI128(ymm_s1, x_s1)
      ymm_s2 = ymm_v10
      VBROADCASTI128(ymm_s2, x_s2)
      ymm_inc = ymm_v14
      VMOVDQA(ymm_inc, mem_inc)
  
      #
      # 4 blocks at a time.
      #
  
      process_2_blocks = Label()
      SUB(reg_blocks, 4)
      JB(process_2_blocks) # < 4 blocks remaining.
  
      VMOVDQA(ymm_v0, ymm_s0)
      VMOVDQA(ymm_v1, ymm_s1)
      VMOVDQA(ymm_v2, ymm_s2)
      VMOVDQA(ymm_v3, ymm_s3)
  
      VMOVDQA(ymm_v4, ymm_v0)
      VMOVDQA(ymm_v5, ymm_v1)
      VMOVDQA(ymm_v6, ymm_v2)
      VPADDQ(ymm_v7, ymm_v3, ymm_inc)
  
      MOV(reg_rounds, 20)
      rounds_loop4 = Loop()
      with rounds_loop4:
          # a += b; d ^= a; d = ROTW16(d);
          ADD_avx2(ymm_v0, ymm_v1)
          ADD_avx2(ymm_v4, ymm_v5)
          XOR_avx2(ymm_v3, ymm_v0)
          XOR_avx2(ymm_v7, ymm_v4)
          ROTW16_avx2(ymm_tmp0, ymm_v3)
          ROTW16_avx2(ymm_tmp0, ymm_v7)
  
          # c += d; b ^= c; b = ROTW12(b);
          ADD_avx2(ymm_v2, ymm_v3)
          ADD_avx2(ymm_v6, ymm_v7)
          XOR_avx2(ymm_v1, ymm_v2)
          XOR_avx2(ymm_v5, ymm_v6)
          ROTW12_avx2(ymm_tmp0, ymm_v1)
          ROTW12_avx2(ymm_tmp0, ymm_v5)
  
          # a += b; d ^= a; d = ROTW8(d);
          ADD_avx2(ymm_v0, ymm_v1)
          ADD_avx2(ymm_v4, ymm_v5)
          XOR_avx2(ymm_v3, ymm_v0)
          XOR_avx2(ymm_v7, ymm_v4)
          ROTW8_avx2(ymm_tmp0, ymm_v3)
          ROTW8_avx2(ymm_tmp0, ymm_v7)
  
          # c += d; b ^= c; b = ROTW7(b)
          ADD_avx2(ymm_v2, ymm_v3)
          ADD_avx2(ymm_v6, ymm_v7)
          XOR_avx2(ymm_v1, ymm_v2)
          XOR_avx2(ymm_v5, ymm_v6)
          ROTW7_avx2(ymm_tmp0, ymm_v1)
          ROTW7_avx2(ymm_tmp0, ymm_v5)
  
          # b = ROTV1(b); c = ROTV2(c);  d = ROTV3(d);
          VPSHUFD(ymm_v1, ymm_v1, 0x39)
          VPSHUFD(ymm_v5, ymm_v5, 0x39)
          VPSHUFD(ymm_v2, ymm_v2, 0x4e)
          VPSHUFD(ymm_v6, ymm_v6, 0x4e)
          VPSHUFD(ymm_v3, ymm_v3, 0x93)
          VPSHUFD(ymm_v7, ymm_v7, 0x93)
  
          # a += b; d ^= a; d = ROTW16(d);
          ADD_avx2(ymm_v0, ymm_v1)
          ADD_avx2(ymm_v4, ymm_v5)
          XOR_avx2(ymm_v3, ymm_v0)
          XOR_avx2(ymm_v7, ymm_v4)
          ROTW16_avx2(ymm_tmp0, ymm_v3)
          ROTW16_avx2(ymm_tmp0, ymm_v7)
  
          # c += d; b ^= c; b = ROTW12(b);
          ADD_avx2(ymm_v2, ymm_v3)
          ADD_avx2(ymm_v6, ymm_v7)
          XOR_avx2(ymm_v1, ymm_v2)
          XOR_avx2(ymm_v5, ymm_v6)
          ROTW12_avx2(ymm_tmp0, ymm_v1)
          ROTW12_avx2(ymm_tmp0, ymm_v5)
  
          # a += b; d ^= a; d = ROTW8(d);
          ADD_avx2(ymm_v0, ymm_v1)
          ADD_avx2(ymm_v4, ymm_v5)
          XOR_avx2(ymm_v3, ymm_v0)
          XOR_avx2(ymm_v7, ymm_v4)
          ROTW8_avx2(ymm_tmp0, ymm_v3)
          ROTW8_avx2(ymm_tmp0, ymm_v7)
  
          # c += d; b ^= c; b = ROTW7(b)
          ADD_avx2(ymm_v2, ymm_v3)
          ADD_avx2(ymm_v6, ymm_v7)
          XOR_avx2(ymm_v1, ymm_v2)
          XOR_avx2(ymm_v5, ymm_v6)
          ROTW7_avx2(ymm_tmp0, ymm_v1)
          ROTW7_avx2(ymm_tmp0, ymm_v5)
  
          # b = ROTV1(b); c = ROTV2(c);  d = ROTV3(d);
          VPSHUFD(ymm_v1, ymm_v1, 0x93)
          VPSHUFD(ymm_v5, ymm_v5, 0x93)
          VPSHUFD(ymm_v2, ymm_v2, 0x4e)
          VPSHUFD(ymm_v6, ymm_v6, 0x4e)
          VPSHUFD(ymm_v3, ymm_v3, 0x39)
          VPSHUFD(ymm_v7, ymm_v7, 0x39)
  
          SUB(reg_rounds, 2)
          JNZ(rounds_loop4.begin)
  
      ADD_avx2(ymm_v0, ymm_s0)
      ADD_avx2(ymm_v1, ymm_s1)
      ADD_avx2(ymm_v2, ymm_s2)
      ADD_avx2(ymm_v3, ymm_s3)
      WriteXor_avx2(ymm_tmp0, reg_inp, reg_outp, 0, ymm_v0, ymm_v1, ymm_v2, ymm_v3)
      ADD_avx2(ymm_s3, ymm_inc)
  
      ADD_avx2(ymm_v4, ymm_s0)
      ADD_avx2(ymm_v5, ymm_s1)
      ADD_avx2(ymm_v6, ymm_s2)
      ADD_avx2(ymm_v7, ymm_s3)
      WriteXor_avx2(ymm_tmp0, reg_inp, reg_outp, 128, ymm_v4, ymm_v5, ymm_v6, ymm_v7)
      ADD_avx2(ymm_s3, ymm_inc)
  
      ADD(reg_inp, 4 * 64)
      ADD(reg_outp, 4 * 64)
      SUB(reg_blocks, 4)
  
      LABEL(process_2_blocks)
      ADD(reg_blocks, 4)
      JZ(out_write_even) # 0 blocks left.
  
      #
      # 2/1 blocks at a time.  The two codepaths are unified because
      # with AVX2 we do 2 blocks at a time anyway, and this only gets called
      # if 3/2/1 blocks are remaining, so the extra branches don't hurt that
      # much.
      #
  
      vector_loop2 = Loop()
      with vector_loop2:
          VMOVDQA(ymm_v0, ymm_s0)
          VMOVDQA(ymm_v1, ymm_s1)
          VMOVDQA(ymm_v2, ymm_s2)
          VMOVDQA(ymm_v3, ymm_s3)
  
          MOV(reg_rounds, 20)
          rounds_loop2 = Loop()
          with rounds_loop2:
              # a += b; d ^= a; d = ROTW16(d);
              ADD_avx2(ymm_v0, ymm_v1)
              XOR_avx2(ymm_v3, ymm_v0)
              ROTW16_avx2(ymm_tmp0, ymm_v3)
  
              # c += d; b ^= c; b = ROTW12(b);
              ADD_avx2(ymm_v2, ymm_v3)
              XOR_avx2(ymm_v1, ymm_v2)
              ROTW12_avx2(ymm_tmp0, ymm_v1)
  
              # a += b; d ^= a; d = ROTW8(d);
              ADD_avx2(ymm_v0, ymm_v1)
              XOR_avx2(ymm_v3, ymm_v0)
              ROTW8_avx2(ymm_tmp0, ymm_v3)
  
              # c += d; b ^= c; b = ROTW7(b)
              ADD_avx2(ymm_v2, ymm_v3)
              XOR_avx2(ymm_v1, ymm_v2)
              ROTW7_avx2(ymm_tmp0, ymm_v1)
  
              # b = ROTV1(b); c = ROTV2(c);  d = ROTV3(d);
              VPSHUFD(ymm_v1, ymm_v1, 0x39)
              VPSHUFD(ymm_v2, ymm_v2, 0x4e)
              VPSHUFD(ymm_v3, ymm_v3, 0x93)
  
              # a += b; d ^= a; d = ROTW16(d);
              ADD_avx2(ymm_v0, ymm_v1)
              XOR_avx2(ymm_v3, ymm_v0)
              ROTW16_avx2(ymm_tmp0, ymm_v3)
  
              # c += d; b ^= c; b = ROTW12(b);
              ADD_avx2(ymm_v2, ymm_v3)
              XOR_avx2(ymm_v1, ymm_v2)
              ROTW12_avx2(ymm_tmp0, ymm_v1)
  
              # a += b; d ^= a; d = ROTW8(d);
              ADD_avx2(ymm_v0, ymm_v1)
              XOR_avx2(ymm_v3, ymm_v0)
              ROTW8_avx2(ymm_tmp0, ymm_v3)
  
              # c += d; b ^= c; b = ROTW7(b)
              ADD_avx2(ymm_v2, ymm_v3)
              XOR_avx2(ymm_v1, ymm_v2)
              ROTW7_avx2(ymm_tmp0, ymm_v1)
  
              # b = ROTV1(b); c = ROTV2(c);  d = ROTV3(d);
              VPSHUFD(ymm_v1, ymm_v1, 0x93)
              VPSHUFD(ymm_v2, ymm_v2, 0x4e)
              VPSHUFD(ymm_v3, ymm_v3, 0x39)
  
              SUB(reg_rounds, 2)
              JNZ(rounds_loop2.begin)
  
          ADD_avx2(ymm_v0, ymm_s0)
          ADD_avx2(ymm_v1, ymm_s1)
          ADD_avx2(ymm_v2, ymm_s2)
          ADD_avx2(ymm_v3, ymm_s3)
  
          # XOR_WRITE(out+ 0, in+ 0, _mm256_permute2x128_si256(v0,v1,0x20));
          VPERM2I128(ymm_tmp0, ymm_v0, ymm_v1, 0x20)
          VPXOR(ymm_tmp0, ymm_tmp0, [reg_inp])
          VMOVDQU([reg_outp], ymm_tmp0)
  
          # XOR_WRITE(out+32, in+32, _mm256_permute2x128_si256(v2,v3,0x20));
          VPERM2I128(ymm_tmp0, ymm_v2, ymm_v3, 0x20)
          VPXOR(ymm_tmp0, ymm_tmp0, [reg_inp+32])
          VMOVDQU([reg_outp+32], ymm_tmp0)
  
          SUB(reg_blocks, 1)
          JZ(out_write_odd)
  
          ADD_avx2(ymm_s3, ymm_inc)
  
          # XOR_WRITE(out+64, in+64, _mm256_permute2x128_si256(v0,v1,0x31));
          VPERM2I128(ymm_tmp0, ymm_v0, ymm_v1, 0x31)
          VPXOR(ymm_tmp0, ymm_tmp0, [reg_inp+64])
          VMOVDQU([reg_outp+64], ymm_tmp0)
  
          # XOR_WRITE(out+96, in+96, _mm256_permute2x128_si256(v2,v3,0x31));
          VPERM2I128(ymm_tmp0, ymm_v2, ymm_v3, 0x31)
          VPXOR(ymm_tmp0, ymm_tmp0, [reg_inp+96])
          VMOVDQU([reg_outp+96], ymm_tmp0)
  
          SUB(reg_blocks, 1)
          JZ(out_write_even)
  
          ADD(reg_inp, 2 * 64)
          ADD(reg_outp, 2 * 64)
          JMP(vector_loop2.begin)
  
      LABEL(out_write_odd)
      VPERM2I128(ymm_s3, ymm_s3, ymm_s3, 0x01) # Odd number of blocks.
  
      LABEL(out_write_even)
      VMOVDQU(x_s3, ymm_s3.as_xmm) # Write back ymm_s3 to x_v3
  
      # Paranoia, cleanse the scratch space.
      VPXOR(ymm_v0, ymm_v0, ymm_v0)
      VMOVDQA(mem_tmp0, ymm_v0)
      VMOVDQA(mem_s3, ymm_v0)
  
      # Clear all YMM (and XMM) registers.
      VZEROALL()
  
      # Remove our stack allocation.
      MOV(registers.rsp, reg_sp_save)
  
      RETURN()
  
  #
  # CPUID
  #
  
  cpuidParams = Argument(ptr(uint32_t))
  
  with Function("cpuidAmd64", (cpuidParams,)):
      reg_params = registers.r15
      LOAD.ARGUMENT(reg_params, cpuidParams)
  
      MOV(registers.eax, [reg_params])
      MOV(registers.ecx, [reg_params+8])
  
      CPUID()
  
      MOV([reg_params], registers.eax)
      MOV([reg_params+4], registers.ebx)
      MOV([reg_params+8], registers.ecx)
      MOV([reg_params+12], registers.edx)
  
      RETURN()
  
  #
  # XGETBV (ECX = 0)
  #
  
  xcrVec = Argument(ptr(uint32_t))
  
  with Function("xgetbv0Amd64", (xcrVec,)):
      reg_vec = GeneralPurposeRegister64()
  
      LOAD.ARGUMENT(reg_vec, xcrVec)
  
      XOR(registers.ecx, registers.ecx)
  
      XGETBV()
  
      MOV([reg_vec], registers.eax)
      MOV([reg_vec+4], registers.edx)
  
      RETURN()
  