text: /* Copyright (c) 2009, 2010, 2011, 2012 ARM Ltd.
  
  Permission is hereby granted, free of charge, to any person obtaining
  a copy of this software and associated documentation files (the
  ``Software''), to deal in the Software without restriction, including
  without limitation the rights to use, copy, modify, merge, publish,
  distribute, sublicense, and/or sell copies of the Software, and to
  permit persons to whom the Software is furnished to do so, subject to
  the following conditions:
  
  The above copyright notice and this permission notice shall be
  included in all copies or substantial portions of the Software.
  
  THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND,
  EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
  CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.  */
  
  #include <stdio.h>
  
  #include <ffi.h>
  #include <ffi_common.h>
  
  #include <stdlib.h>
  
  /* Stack alignment requirement in bytes */
  #if defined (__APPLE__)
  #define AARCH64_STACK_ALIGN 1
  #else
  #define AARCH64_STACK_ALIGN 16
  #endif
  
  #define N_X_ARG_REG 8
  #define N_V_ARG_REG 8
  
  #define AARCH64_FFI_WITH_V (1 << AARCH64_FFI_WITH_V_BIT)
  
  union _d
  {
    UINT64 d;
    UINT32 s[2];
  };
  
  struct call_context
  {
    UINT64 x [AARCH64_N_XREG];
    struct
    {
      union _d d[2];
    } v [AARCH64_N_VREG];
  };
  
  #if defined (__clang__) && defined (__APPLE__)
  extern void
  sys_icache_invalidate (void *start, size_t len);
  #endif
  
  static inline void
  ffi_clear_cache (void *start, void *end)
  {
  #if defined (__clang__) && defined (__APPLE__)
  	sys_icache_invalidate (start, (char *)end - (char *)start);
  #elif defined (__GNUC__)
  	__builtin___clear_cache (start, end);
  #else
  #error "Missing builtin to flush instruction cache"
  #endif
  }
  
  static void *
  get_x_addr (struct call_context *context, unsigned n)
  {
    return &context->x[n];
  }
  
  static void *
  get_s_addr (struct call_context *context, unsigned n)
  {
  #if defined __AARCH64EB__
    return &context->v[n].d[1].s[1];
  #else
    return &context->v[n].d[0].s[0];
  #endif
  }
  
  static void *
  get_d_addr (struct call_context *context, unsigned n)
  {
  #if defined __AARCH64EB__
    return &context->v[n].d[1];
  #else
    return &context->v[n].d[0];
  #endif
  }
  
  static void *
  get_v_addr (struct call_context *context, unsigned n)
  {
    return &context->v[n];
  }
  
  /* Return the memory location at which a basic type would reside
     were it to have been stored in register n.  */
  
  static void *
  get_basic_type_addr (unsigned short type, struct call_context *context,
  		     unsigned n)
  {
    switch (type)
      {
      case FFI_TYPE_FLOAT:
        return get_s_addr (context, n);
      case FFI_TYPE_DOUBLE:
        return get_d_addr (context, n);
  #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
      case FFI_TYPE_LONGDOUBLE:
        return get_v_addr (context, n);
  #endif
      case FFI_TYPE_UINT8:
      case FFI_TYPE_SINT8:
      case FFI_TYPE_UINT16:
      case FFI_TYPE_SINT16:
      case FFI_TYPE_UINT32:
      case FFI_TYPE_SINT32:
      case FFI_TYPE_INT:
      case FFI_TYPE_POINTER:
      case FFI_TYPE_UINT64:
      case FFI_TYPE_SINT64:
        return get_x_addr (context, n);
      case FFI_TYPE_VOID:
        return NULL;
      default:
        FFI_ASSERT (0);
        return NULL;
      }
  }
  
  /* Return the alignment width for each of the basic types.  */
  
  static size_t
  get_basic_type_alignment (unsigned short type)
  {
    switch (type)
      {
      case FFI_TYPE_FLOAT:
  #if defined (__APPLE__)
        return sizeof (UINT32);
  #endif
      case FFI_TYPE_DOUBLE:
        return sizeof (UINT64);
  #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
      case FFI_TYPE_LONGDOUBLE:
        return sizeof (long double);
  #endif
      case FFI_TYPE_UINT8:
      case FFI_TYPE_SINT8:
  #if defined (__APPLE__)
  	  return sizeof (UINT8);
  #endif
      case FFI_TYPE_UINT16:
      case FFI_TYPE_SINT16:
  #if defined (__APPLE__)
  	  return sizeof (UINT16);
  #endif
      case FFI_TYPE_UINT32:
      case FFI_TYPE_INT:
      case FFI_TYPE_SINT32:
  #if defined (__APPLE__)
  	  return sizeof (UINT32);
  #endif
      case FFI_TYPE_POINTER:
      case FFI_TYPE_UINT64:
      case FFI_TYPE_SINT64:
        return sizeof (UINT64);
  
      default:
        FFI_ASSERT (0);
        return 0;
      }
  }
  
  /* Return the size in bytes for each of the basic types.  */
  
  static size_t
  get_basic_type_size (unsigned short type)
  {
    switch (type)
      {
      case FFI_TYPE_FLOAT:
        return sizeof (UINT32);
      case FFI_TYPE_DOUBLE:
        return sizeof (UINT64);
  #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
      case FFI_TYPE_LONGDOUBLE:
        return sizeof (long double);
  #endif
      case FFI_TYPE_UINT8:
        return sizeof (UINT8);
      case FFI_TYPE_SINT8:
        return sizeof (SINT8);
      case FFI_TYPE_UINT16:
        return sizeof (UINT16);
      case FFI_TYPE_SINT16:
        return sizeof (SINT16);
      case FFI_TYPE_UINT32:
        return sizeof (UINT32);
      case FFI_TYPE_INT:
      case FFI_TYPE_SINT32:
        return sizeof (SINT32);
      case FFI_TYPE_POINTER:
      case FFI_TYPE_UINT64:
        return sizeof (UINT64);
      case FFI_TYPE_SINT64:
        return sizeof (SINT64);
  
      default:
        FFI_ASSERT (0);
        return 0;
      }
  }
  
  extern void
  ffi_call_SYSV (unsigned (*)(struct call_context *context, unsigned char *,
  			    extended_cif *),
                 struct call_context *context,
                 extended_cif *,
                 size_t,
                 void (*fn)(void));
  
  extern void
  ffi_closure_SYSV (ffi_closure *);
  
  /* Test for an FFI floating point representation.  */
  
  static unsigned
  is_floating_type (unsigned short type)
  {
    return (type == FFI_TYPE_FLOAT || type == FFI_TYPE_DOUBLE
  	  || type == FFI_TYPE_LONGDOUBLE);
  }
  
  /* Test for a homogeneous structure.  */
  
  static unsigned short
  get_homogeneous_type (ffi_type *ty)
  {
    if (ty->type == FFI_TYPE_STRUCT && ty->elements)
      {
        unsigned i;
        unsigned short candidate_type
  	= get_homogeneous_type (ty->elements[0]);
        for (i =1; ty->elements[i]; i++)
  	{
  	  unsigned short iteration_type = 0;
  	  /* If we have a nested struct, we must find its homogeneous type.
  	     If that fits with our candidate type, we are still
  	     homogeneous.  */
  	  if (ty->elements[i]->type == FFI_TYPE_STRUCT
  	      && ty->elements[i]->elements)
  	    {
  	      iteration_type = get_homogeneous_type (ty->elements[i]);
  	    }
  	  else
  	    {
  	      iteration_type = ty->elements[i]->type;
  	    }
  
  	  /* If we are not homogeneous, return FFI_TYPE_STRUCT.  */
  	  if (candidate_type != iteration_type)
  	    return FFI_TYPE_STRUCT;
  	}
        return candidate_type;
      }
  
    /* Base case, we have no more levels of nesting, so we
       are a basic type, and so, trivially homogeneous in that type.  */
    return ty->type;
  }
  
  /* Determine the number of elements within a STRUCT.
  
     Note, we must handle nested structs.
  
     If ty is not a STRUCT this function will return 0.  */
  
  static unsigned
  element_count (ffi_type *ty)
  {
    if (ty->type == FFI_TYPE_STRUCT && ty->elements)
      {
        unsigned n;
        unsigned elems = 0;
        for (n = 0; ty->elements[n]; n++)
  	{
  	  if (ty->elements[n]->type == FFI_TYPE_STRUCT
  	      && ty->elements[n]->elements)
  	    elems += element_count (ty->elements[n]);
  	  else
  	    elems++;
  	}
        return elems;
      }
    return 0;
  }
  
  /* Test for a homogeneous floating point aggregate.
  
     A homogeneous floating point aggregate is a homogeneous aggregate of
     a half- single- or double- precision floating point type with one
     to four elements.  Note that this includes nested structs of the
     basic type.  */
  
  static int
  is_hfa (ffi_type *ty)
  {
    if (ty->type == FFI_TYPE_STRUCT
        && ty->elements[0]
        && is_floating_type (get_homogeneous_type (ty)))
      {
        unsigned n = element_count (ty);
        return n >= 1 && n <= 4;
      }
    return 0;
  }
  
  /* Test if an ffi_type is a candidate for passing in a register.
  
     This test does not check that sufficient registers of the
     appropriate class are actually available, merely that IFF
     sufficient registers are available then the argument will be passed
     in register(s).
  
     Note that an ffi_type that is deemed to be a register candidate
     will always be returned in registers.
  
     Returns 1 if a register candidate else 0.  */
  
  static int
  is_register_candidate (ffi_type *ty)
  {
    switch (ty->type)
      {
      case FFI_TYPE_VOID:
      case FFI_TYPE_FLOAT:
      case FFI_TYPE_DOUBLE:
  #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
      case FFI_TYPE_LONGDOUBLE:
  #endif
      case FFI_TYPE_UINT8:
      case FFI_TYPE_UINT16:
      case FFI_TYPE_UINT32:
      case FFI_TYPE_UINT64:
      case FFI_TYPE_POINTER:
      case FFI_TYPE_SINT8:
      case FFI_TYPE_SINT16:
      case FFI_TYPE_SINT32:
      case FFI_TYPE_INT:
      case FFI_TYPE_SINT64:
        return 1;
  
      case FFI_TYPE_STRUCT:
        if (is_hfa (ty))
          {
            return 1;
          }
        else if (ty->size > 16)
          {
            /* Too large. Will be replaced with a pointer to memory. The
               pointer MAY be passed in a register, but the value will
               not. This test specifically fails since the argument will
               never be passed by value in registers. */
            return 0;
          }
        else
          {
            /* Might be passed in registers depending on the number of
               registers required. */
            return (ty->size + 7) / 8 < N_X_ARG_REG;
          }
        break;
  
      default:
        FFI_ASSERT (0);
        break;
      }
  
    return 0;
  }
  
  /* Test if an ffi_type argument or result is a candidate for a vector
     register.  */
  
  static int
  is_v_register_candidate (ffi_type *ty)
  {
    return is_floating_type (ty->type)
  	   || (ty->type == FFI_TYPE_STRUCT && is_hfa (ty));
  }
  
  /* Representation of the procedure call argument marshalling
     state.
  
     The terse state variable names match the names used in the AARCH64
     PCS. */
  
  struct arg_state
  {
    unsigned ngrn;                /* Next general-purpose register number. */
    unsigned nsrn;                /* Next vector register number. */
    size_t nsaa;                  /* Next stack offset. */
  
  #if defined (__APPLE__)
    unsigned allocating_variadic;
  #endif
  };
  
  /* Initialize a procedure call argument marshalling state.  */
  static void
  arg_init (struct arg_state *state, size_t call_frame_size)
  {
    state->ngrn = 0;
    state->nsrn = 0;
    state->nsaa = 0;
  
  #if defined (__APPLE__)
    state->allocating_variadic = 0;
  #endif
  }
  
  /* Return the number of available consecutive core argument
     registers.  */
  
  static unsigned
  available_x (struct arg_state *state)
  {
    return N_X_ARG_REG - state->ngrn;
  }
  
  /* Return the number of available consecutive vector argument
     registers.  */
  
  static unsigned
  available_v (struct arg_state *state)
  {
    return N_V_ARG_REG - state->nsrn;
  }
  
  static void *
  allocate_to_x (struct call_context *context, struct arg_state *state)
  {
    FFI_ASSERT (state->ngrn < N_X_ARG_REG);
    return get_x_addr (context, (state->ngrn)++);
  }
  
  static void *
  allocate_to_s (struct call_context *context, struct arg_state *state)
  {
    FFI_ASSERT (state->nsrn < N_V_ARG_REG);
    return get_s_addr (context, (state->nsrn)++);
  }
  
  static void *
  allocate_to_d (struct call_context *context, struct arg_state *state)
  {
    FFI_ASSERT (state->nsrn < N_V_ARG_REG);
    return get_d_addr (context, (state->nsrn)++);
  }
  
  static void *
  allocate_to_v (struct call_context *context, struct arg_state *state)
  {
    FFI_ASSERT (state->nsrn < N_V_ARG_REG);
    return get_v_addr (context, (state->nsrn)++);
  }
  
  /* Allocate an aligned slot on the stack and return a pointer to it.  */
  static void *
  allocate_to_stack (struct arg_state *state, void *stack, size_t alignment,
  		   size_t size)
  {
    void *allocation;
  
    /* Round up the NSAA to the larger of 8 or the natural
       alignment of the argument's type.  */
    state->nsaa = ALIGN (state->nsaa, alignment);
    state->nsaa = ALIGN (state->nsaa, alignment);
  #if defined (__APPLE__)
    if (state->allocating_variadic)
      state->nsaa = ALIGN (state->nsaa, 8);
  #else
    state->nsaa = ALIGN (state->nsaa, 8);
  #endif
  
    allocation = stack + state->nsaa;
  
    state->nsaa += size;
    return allocation;
  }
  
  static void
  copy_basic_type (void *dest, void *source, unsigned short type)
  {
    /* This is necessary to ensure that basic types are copied
       sign extended to 64-bits as libffi expects.  */
    switch (type)
      {
      case FFI_TYPE_FLOAT:
        *(float *) dest = *(float *) source;
        break;
      case FFI_TYPE_DOUBLE:
        *(double *) dest = *(double *) source;
        break;
  #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
      case FFI_TYPE_LONGDOUBLE:
        *(long double *) dest = *(long double *) source;
        break;
  #endif
      case FFI_TYPE_UINT8:
        *(ffi_arg *) dest = *(UINT8 *) source;
        break;
      case FFI_TYPE_SINT8:
        *(ffi_sarg *) dest = *(SINT8 *) source;
        break;
      case FFI_TYPE_UINT16:
        *(ffi_arg *) dest = *(UINT16 *) source;
        break;
      case FFI_TYPE_SINT16:
        *(ffi_sarg *) dest = *(SINT16 *) source;
        break;
      case FFI_TYPE_UINT32:
        *(ffi_arg *) dest = *(UINT32 *) source;
        break;
      case FFI_TYPE_INT:
      case FFI_TYPE_SINT32:
        *(ffi_sarg *) dest = *(SINT32 *) source;
        break;
      case FFI_TYPE_POINTER:
      case FFI_TYPE_UINT64:
        *(ffi_arg *) dest = *(UINT64 *) source;
        break;
      case FFI_TYPE_SINT64:
        *(ffi_sarg *) dest = *(SINT64 *) source;
        break;
      case FFI_TYPE_VOID:
        break;
  
      default:
        FFI_ASSERT (0);
      }
  }
  
  static void
  copy_hfa_to_reg_or_stack (void *memory,
  			  ffi_type *ty,
  			  struct call_context *context,
  			  unsigned char *stack,
  			  struct arg_state *state)
  {
    unsigned elems = element_count (ty);
    if (available_v (state) < elems)
      {
        /* There are insufficient V registers. Further V register allocations
  	 are prevented, the NSAA is adjusted (by allocate_to_stack ())
  	 and the argument is copied to memory at the adjusted NSAA.  */
        state->nsrn = N_V_ARG_REG;
        memcpy (allocate_to_stack (state, stack, ty->alignment, ty->size),
  	      memory,
  	      ty->size);
      }
    else
      {
        int i;
        unsigned short type = get_homogeneous_type (ty);
        for (i = 0; i < elems; i++)
  	{
  	  void *reg = allocate_to_v (context, state);
  	  copy_basic_type (reg, memory, type);
  	  memory += get_basic_type_size (type);
  	}
      }
  }
  
  /* Either allocate an appropriate register for the argument type, or if
     none are available, allocate a stack slot and return a pointer
     to the allocated space.  */
  
  static void *
  allocate_to_register_or_stack (struct call_context *context,
  			       unsigned char *stack,
  			       struct arg_state *state,
  			       unsigned short type)
  {
    size_t alignment = get_basic_type_alignment (type);
    size_t size = alignment;
    switch (type)
      {
      case FFI_TYPE_FLOAT:
        /* This is the only case for which the allocated stack size
  	 should not match the alignment of the type.  */
        size = sizeof (UINT32);
        /* Fall through.  */
      case FFI_TYPE_DOUBLE:
        if (state->nsrn < N_V_ARG_REG)
  	return allocate_to_d (context, state);
        state->nsrn = N_V_ARG_REG;
        break;
  #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
      case FFI_TYPE_LONGDOUBLE:
        if (state->nsrn < N_V_ARG_REG)
  	return allocate_to_v (context, state);
        state->nsrn = N_V_ARG_REG;
        break;
  #endif
      case FFI_TYPE_UINT8:
      case FFI_TYPE_SINT8:
      case FFI_TYPE_UINT16:
      case FFI_TYPE_SINT16:
      case FFI_TYPE_UINT32:
      case FFI_TYPE_SINT32:
      case FFI_TYPE_INT:
      case FFI_TYPE_POINTER:
      case FFI_TYPE_UINT64:
      case FFI_TYPE_SINT64:
        if (state->ngrn < N_X_ARG_REG)
  	return allocate_to_x (context, state);
        state->ngrn = N_X_ARG_REG;
        break;
      default:
        FFI_ASSERT (0);
      }
  
      return allocate_to_stack (state, stack, alignment, size);
  }
  
  /* Copy a value to an appropriate register, or if none are
     available, to the stack.  */
  
  static void
  copy_to_register_or_stack (struct call_context *context,
  			   unsigned char *stack,
  			   struct arg_state *state,
  			   void *value,
  			   unsigned short type)
  {
    copy_basic_type (
  	  allocate_to_register_or_stack (context, stack, state, type),
  	  value,
  	  type);
  }
  
  /* Marshall the arguments from FFI representation to procedure call
     context and stack.  */
  
  static unsigned
  aarch64_prep_args (struct call_context *context, unsigned char *stack,
  		   extended_cif *ecif)
  {
    int i;
    struct arg_state state;
  
    arg_init (&state, ALIGN(ecif->cif->bytes, 16));
  
    for (i = 0; i < ecif->cif->nargs; i++)
      {
        ffi_type *ty = ecif->cif->arg_types[i];
        switch (ty->type)
  	{
  	case FFI_TYPE_VOID:
  	  FFI_ASSERT (0);
  	  break;
  
  	/* If the argument is a basic type the argument is allocated to an
  	   appropriate register, or if none are available, to the stack.  */
  	case FFI_TYPE_FLOAT:
  	case FFI_TYPE_DOUBLE:
  #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
  	case FFI_TYPE_LONGDOUBLE:
  #endif
  	case FFI_TYPE_UINT8:
  	case FFI_TYPE_SINT8:
  	case FFI_TYPE_UINT16:
  	case FFI_TYPE_SINT16:
  	case FFI_TYPE_UINT32:
  	case FFI_TYPE_INT:
  	case FFI_TYPE_SINT32:
  	case FFI_TYPE_POINTER:
  	case FFI_TYPE_UINT64:
  	case FFI_TYPE_SINT64:
  	  copy_to_register_or_stack (context, stack, &state,
  				     ecif->avalue[i], ty->type);
  	  break;
  
  	case FFI_TYPE_STRUCT:
  	  if (is_hfa (ty))
  	    {
  	      copy_hfa_to_reg_or_stack (ecif->avalue[i], ty, context,
  					stack, &state);
  	    }
  	  else if (ty->size > 16)
  	    {
  	      /* If the argument is a composite type that is larger than 16
  		 bytes, then the argument has been copied to memory, and
  		 the argument is replaced by a pointer to the copy.  */
  
  	      copy_to_register_or_stack (context, stack, &state,
  					 &(ecif->avalue[i]), FFI_TYPE_POINTER);
  	    }
  	  else if (available_x (&state) >= (ty->size + 7) / 8)
  	    {
  	      /* If the argument is a composite type and the size in
  		 double-words is not more than the number of available
  		 X registers, then the argument is copied into consecutive
  		 X registers.  */
  	      int j;
  	      for (j = 0; j < (ty->size + 7) / 8; j++)
  		{
  		  memcpy (allocate_to_x (context, &state),
  			  &(((UINT64 *) ecif->avalue[i])[j]),
  			  sizeof (UINT64));
  		}
  	    }
  	  else
  	    {
  	      /* Otherwise, there are insufficient X registers. Further X
  		 register allocations are prevented, the NSAA is adjusted
  		 (by allocate_to_stack ()) and the argument is copied to
  		 memory at the adjusted NSAA.  */
  	      state.ngrn = N_X_ARG_REG;
  
  	      memcpy (allocate_to_stack (&state, stack, ty->alignment,
  					 ty->size), ecif->avalue + i, ty->size);
  	    }
  	  break;
  
  	default:
  	  FFI_ASSERT (0);
  	  break;
  	}
  
  #if defined (__APPLE__)
        if (i + 1 == ecif->cif->aarch64_nfixedargs)
  	{
  	  state.ngrn = N_X_ARG_REG;
  	  state.nsrn = N_V_ARG_REG;
  
  	  state.allocating_variadic = 1;
  	}
  #endif
      }
  
    return ecif->cif->aarch64_flags;
  }
  
  ffi_status
  ffi_prep_cif_machdep (ffi_cif *cif)
  {
    /* Round the stack up to a multiple of the stack alignment requirement. */
    cif->bytes =
      (cif->bytes + (AARCH64_STACK_ALIGN - 1)) & ~ (AARCH64_STACK_ALIGN - 1);
  
    /* Initialize our flags. We are interested if this CIF will touch a
       vector register, if so we will enable context save and load to
       those registers, otherwise not. This is intended to be friendly
       to lazy float context switching in the kernel.  */
    cif->aarch64_flags = 0;
  
    if (is_v_register_candidate (cif->rtype))
      {
        cif->aarch64_flags |= AARCH64_FFI_WITH_V;
      }
    else
      {
        int i;
        for (i = 0; i < cif->nargs; i++)
          if (is_v_register_candidate (cif->arg_types[i]))
            {
              cif->aarch64_flags |= AARCH64_FFI_WITH_V;
              break;
            }
      }
  
  #if defined (__APPLE__)
    cif->aarch64_nfixedargs = 0;
  #endif
  
    return FFI_OK;
  }
  
  #if defined (__APPLE__)
  
  /* Perform Apple-specific cif processing for variadic calls */
  ffi_status ffi_prep_cif_machdep_var(ffi_cif *cif,
  				    unsigned int nfixedargs,
  				    unsigned int ntotalargs)
  {
    ffi_status status;
  
    status = ffi_prep_cif_machdep (cif);
  
    cif->aarch64_nfixedargs = nfixedargs;
  
    return status;
  }
  
  #endif
  
  /* Call a function with the provided arguments and capture the return
     value.  */
  void
  ffi_call (ffi_cif *cif, void (*fn)(void), void *rvalue, void **avalue)
  {
    extended_cif ecif;
  
    ecif.cif = cif;
    ecif.avalue = avalue;
    ecif.rvalue = rvalue;
  
    switch (cif->abi)
      {
      case FFI_SYSV:
        {
          struct call_context context;
  	size_t stack_bytes;
  
  	/* Figure out the total amount of stack space we need, the
  	   above call frame space needs to be 16 bytes aligned to
  	   ensure correct alignment of the first object inserted in
  	   that space hence the ALIGN applied to cif->bytes.*/
  	stack_bytes = ALIGN(cif->bytes, 16);
  
  	memset (&context, 0, sizeof (context));
          if (is_register_candidate (cif->rtype))
            {
              ffi_call_SYSV (aarch64_prep_args, &context, &ecif, stack_bytes, fn);
              switch (cif->rtype->type)
                {
                case FFI_TYPE_VOID:
                case FFI_TYPE_FLOAT:
                case FFI_TYPE_DOUBLE:
  #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
                case FFI_TYPE_LONGDOUBLE:
  #endif
                case FFI_TYPE_UINT8:
                case FFI_TYPE_SINT8:
                case FFI_TYPE_UINT16:
                case FFI_TYPE_SINT16:
                case FFI_TYPE_UINT32:
                case FFI_TYPE_SINT32:
                case FFI_TYPE_POINTER:
                case FFI_TYPE_UINT64:
                case FFI_TYPE_INT:
                case FFI_TYPE_SINT64:
  		{
  		  void *addr = get_basic_type_addr (cif->rtype->type,
  						    &context, 0);
  		  copy_basic_type (rvalue, addr, cif->rtype->type);
  		  break;
  		}
  
                case FFI_TYPE_STRUCT:
                  if (is_hfa (cif->rtype))
  		  {
  		    int j;
  		    unsigned short type = get_homogeneous_type (cif->rtype);
  		    unsigned elems = element_count (cif->rtype);
  		    for (j = 0; j < elems; j++)
  		      {
  			void *reg = get_basic_type_addr (type, &context, j);
  			copy_basic_type (rvalue, reg, type);
  			rvalue += get_basic_type_size (type);
  		      }
  		  }
                  else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG)
                    {
                      size_t size = ALIGN (cif->rtype->size, sizeof (UINT64));
                      memcpy (rvalue, get_x_addr (&context, 0), size);
                    }
                  else
                    {
                      FFI_ASSERT (0);
                    }
                  break;
  
                default:
                  FFI_ASSERT (0);
                  break;
                }
            }
          else
            {
              memcpy (get_x_addr (&context, 8), &rvalue, sizeof (UINT64));
              ffi_call_SYSV (aarch64_prep_args, &context, &ecif,
  			   stack_bytes, fn);
            }
          break;
        }
  
      default:
        FFI_ASSERT (0);
        break;
      }
  }
  
  static unsigned char trampoline [] =
  { 0x70, 0x00, 0x00, 0x58,	/* ldr	x16, 1f	*/
    0x91, 0x00, 0x00, 0x10,	/* adr	x17, 2f	*/
    0x00, 0x02, 0x1f, 0xd6	/* br	x16	*/
  };
  
  /* Build a trampoline.  */
  
  #define FFI_INIT_TRAMPOLINE(TRAMP,FUN,CTX,FLAGS)			\
    ({unsigned char *__tramp = (unsigned char*)(TRAMP);			\
      UINT64  __fun = (UINT64)(FUN);					\
      UINT64  __ctx = (UINT64)(CTX);					\
      UINT64  __flags = (UINT64)(FLAGS);					\
      memcpy (__tramp, trampoline, sizeof (trampoline));			\
      memcpy (__tramp + 12, &__fun, sizeof (__fun));			\
      memcpy (__tramp + 20, &__ctx, sizeof (__ctx));			\
      memcpy (__tramp + 28, &__flags, sizeof (__flags));			\
      ffi_clear_cache(__tramp, __tramp + FFI_TRAMPOLINE_SIZE);		\
    })
  
  ffi_status
  ffi_prep_closure_loc (ffi_closure* closure,
                        ffi_cif* cif,
                        void (*fun)(ffi_cif*,void*,void**,void*),
                        void *user_data,
                        void *codeloc)
  {
    if (cif->abi != FFI_SYSV)
      return FFI_BAD_ABI;
  
    FFI_INIT_TRAMPOLINE (&closure->tramp[0], &ffi_closure_SYSV, codeloc,
  		       cif->aarch64_flags);
  
    closure->cif  = cif;
    closure->user_data = user_data;
    closure->fun  = fun;
  
    return FFI_OK;
  }
  
  /* Primary handler to setup and invoke a function within a closure.
  
     A closure when invoked enters via the assembler wrapper
     ffi_closure_SYSV(). The wrapper allocates a call context on the
     stack, saves the interesting registers (from the perspective of
     the calling convention) into the context then passes control to
     ffi_closure_SYSV_inner() passing the saved context and a pointer to
     the stack at the point ffi_closure_SYSV() was invoked.
  
     On the return path the assembler wrapper will reload call context
     registers.
  
     ffi_closure_SYSV_inner() marshalls the call context into ffi value
     descriptors, invokes the wrapped function, then marshalls the return
     value back into the call context.  */
  
  void FFI_HIDDEN
  ffi_closure_SYSV_inner (ffi_closure *closure, struct call_context *context,
  			void *stack)
  {
    ffi_cif *cif = closure->cif;
    void **avalue = (void**) alloca (cif->nargs * sizeof (void*));
    void *rvalue = NULL;
    int i;
    struct arg_state state;
  
    arg_init (&state, ALIGN(cif->bytes, 16));
  
    for (i = 0; i < cif->nargs; i++)
      {
        ffi_type *ty = cif->arg_types[i];
  
        switch (ty->type)
  	{
  	case FFI_TYPE_VOID:
  	  FFI_ASSERT (0);
  	  break;
  
  	case FFI_TYPE_UINT8:
  	case FFI_TYPE_SINT8:
  	case FFI_TYPE_UINT16:
  	case FFI_TYPE_SINT16:
  	case FFI_TYPE_UINT32:
  	case FFI_TYPE_SINT32:
  	case FFI_TYPE_INT:
  	case FFI_TYPE_POINTER:
  	case FFI_TYPE_UINT64:
  	case FFI_TYPE_SINT64:
  	case  FFI_TYPE_FLOAT:
  	case  FFI_TYPE_DOUBLE:
  #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
  	case  FFI_TYPE_LONGDOUBLE:
  	  avalue[i] = allocate_to_register_or_stack (context, stack,
  						     &state, ty->type);
  	  break;
  #endif
  
  	case FFI_TYPE_STRUCT:
  	  if (is_hfa (ty))
  	    {
  	      unsigned n = element_count (ty);
  	      if (available_v (&state) < n)
  		{
  		  state.nsrn = N_V_ARG_REG;
  		  avalue[i] = allocate_to_stack (&state, stack, ty->alignment,
  						 ty->size);
  		}
  	      else
  		{
  		  switch (get_homogeneous_type (ty))
  		    {
  		    case FFI_TYPE_FLOAT:
  		      {
  			/* Eeek! We need a pointer to the structure,
  			   however the homogeneous float elements are
  			   being passed in individual S registers,
  			   therefore the structure is not represented as
  			   a contiguous sequence of bytes in our saved
  			   register context. We need to fake up a copy
  			   of the structure laid out in memory
  			   correctly. The fake can be tossed once the
  			   closure function has returned hence alloca()
  			   is sufficient. */
  			int j;
  			UINT32 *p = avalue[i] = alloca (ty->size);
  			for (j = 0; j < element_count (ty); j++)
  			  memcpy (&p[j],
  				  allocate_to_s (context, &state),
  				  sizeof (*p));
  			break;
  		      }
  
  		    case FFI_TYPE_DOUBLE:
  		      {
  			/* Eeek! We need a pointer to the structure,
  			   however the homogeneous float elements are
  			   being passed in individual S registers,
  			   therefore the structure is not represented as
  			   a contiguous sequence of bytes in our saved
  			   register context. We need to fake up a copy
  			   of the structure laid out in memory
  			   correctly. The fake can be tossed once the
  			   closure function has returned hence alloca()
  			   is sufficient. */
  			int j;
  			UINT64 *p = avalue[i] = alloca (ty->size);
  			for (j = 0; j < element_count (ty); j++)
  			  memcpy (&p[j],
  				  allocate_to_d (context, &state),
  				  sizeof (*p));
  			break;
  		      }
  
  #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
  		    case FFI_TYPE_LONGDOUBLE:
  			  memcpy (&avalue[i],
  				  allocate_to_v (context, &state),
  				  sizeof (*avalue));
  		      break;
  #endif
  
  		    default:
  		      FFI_ASSERT (0);
  		      break;
  		    }
  		}
  	    }
  	  else if (ty->size > 16)
  	    {
  	      /* Replace Composite type of size greater than 16 with a
  		 pointer.  */
  	      memcpy (&avalue[i],
  		      allocate_to_register_or_stack (context, stack,
  						     &state, FFI_TYPE_POINTER),
  		      sizeof (avalue[i]));
  	    }
  	  else if (available_x (&state) >= (ty->size + 7) / 8)
  	    {
  	      avalue[i] = get_x_addr (context, state.ngrn);
  	      state.ngrn += (ty->size + 7) / 8;
  	    }
  	  else
  	    {
  	      state.ngrn = N_X_ARG_REG;
  
  	      avalue[i] = allocate_to_stack (&state, stack, ty->alignment,
  					     ty->size);
  	    }
  	  break;
  
  	default:
  	  FFI_ASSERT (0);
  	  break;
  	}
      }
  
    /* Figure out where the return value will be passed, either in
       registers or in a memory block allocated by the caller and passed
       in x8.  */
  
    if (is_register_candidate (cif->rtype))
      {
        /* Register candidates are *always* returned in registers. */
  
        /* Allocate a scratchpad for the return value, we will let the
           callee scrible the result into the scratch pad then move the
           contents into the appropriate return value location for the
           call convention.  */
        rvalue = alloca (cif->rtype->size);
        (closure->fun) (cif, rvalue, avalue, closure->user_data);
  
        /* Copy the return value into the call context so that it is returned
           as expected to our caller.  */
        switch (cif->rtype->type)
          {
          case FFI_TYPE_VOID:
            break;
  
          case FFI_TYPE_UINT8:
          case FFI_TYPE_UINT16:
          case FFI_TYPE_UINT32:
          case FFI_TYPE_POINTER:
          case FFI_TYPE_UINT64:
          case FFI_TYPE_SINT8:
          case FFI_TYPE_SINT16:
          case FFI_TYPE_INT:
          case FFI_TYPE_SINT32:
          case FFI_TYPE_SINT64:
          case FFI_TYPE_FLOAT:
          case FFI_TYPE_DOUBLE:
  #if FFI_TYPE_DOUBLE != FFI_TYPE_LONGDOUBLE
          case FFI_TYPE_LONGDOUBLE:
  #endif
  	  {
  	    void *addr = get_basic_type_addr (cif->rtype->type, context, 0);
  	    copy_basic_type (addr, rvalue, cif->rtype->type);
              break;
  	  }
          case FFI_TYPE_STRUCT:
            if (is_hfa (cif->rtype))
  	    {
  	      int j;
  	      unsigned short type = get_homogeneous_type (cif->rtype);
  	      unsigned elems = element_count (cif->rtype);
  	      for (j = 0; j < elems; j++)
  		{
  		  void *reg = get_basic_type_addr (type, context, j);
  		  copy_basic_type (reg, rvalue, type);
  		  rvalue += get_basic_type_size (type);
  		}
  	    }
            else if ((cif->rtype->size + 7) / 8 < N_X_ARG_REG)
              {
                size_t size = ALIGN (cif->rtype->size, sizeof (UINT64)) ;
                memcpy (get_x_addr (context, 0), rvalue, size);
              }
            else
              {
                FFI_ASSERT (0);
              }
            break;
          default:
            FFI_ASSERT (0);
            break;
          }
      }
    else
      {
        memcpy (&rvalue, get_x_addr (context, 8), sizeof (UINT64));
        (closure->fun) (cif, rvalue, avalue, closure->user_data);
      }
  }
  
  