text: /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
  /*
   *  (C) 2001 by Argonne National Laboratory.
   *      See COPYRIGHT in top-level directory.
   */
  
  #include "mpidimpl.h"
  
  /* This file contains two types of routines associated with requests: 
   * Routines to allocate and free requests
   * Routines to manage iovs on requests 
   *
   * Note that there are a number of macros that also manage requests defined
   * in mpidimpl.h ; these are intended to optimize request creation for
   * specific types of requests.  See the comments in mpidimpl.h for more
   * details.
   */
  
  /* Routines and data structures for request allocation and deallocation */
  #ifndef MPID_REQUEST_PREALLOC
  #define MPID_REQUEST_PREALLOC 8
  #endif
  
  /* Max depth of recursive calls of MPID_Request_complete */
  #define REQUEST_CB_DEPTH 2
  #define MPIDI_LOAD_RECV_IOV_ORIG_SEGMENT_FIRST_UNSET (-1)
  
  MPID_Request MPID_Request_direct[MPID_REQUEST_PREALLOC] = {{0}};
  MPIU_Object_alloc_t MPID_Request_mem = {
      0, 0, 0, 0, MPID_REQUEST, sizeof(MPID_Request), MPID_Request_direct,
      MPID_REQUEST_PREALLOC };
  
  /* See the comments above about request creation.  Some routines will
     use macros in mpidimpl.h *instead* of this routine */
  #undef FUNCNAME
  #define FUNCNAME MPID_Request_create
  #undef FCNAME
  #define FCNAME MPL_QUOTE(FUNCNAME)
  MPID_Request * MPID_Request_create(void)
  {
      MPID_Request * req;
      MPIDI_STATE_DECL(MPID_STATE_MPID_REQUEST_CREATE);
  
      MPIDI_FUNC_ENTER(MPID_STATE_MPID_REQUEST_CREATE);
      
      req = MPIU_Handle_obj_alloc(&MPID_Request_mem);
      if (req != NULL)
      {
  	MPIU_DBG_MSG_P(CH3_CHANNEL,VERBOSE,
  		       "allocated request, handle=0x%08x", req->handle);
  #ifdef MPICH_DBG_OUTPUT
  	/*MPIU_Assert(HANDLE_GET_MPI_KIND(req->handle) == MPID_REQUEST);*/
  	if (HANDLE_GET_MPI_KIND(req->handle) != MPID_REQUEST)
  	{
  	    int mpi_errno;
  	    mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, 
  		       FCNAME, __LINE__, MPI_ERR_OTHER, 
  		       "**invalid_handle", "**invalid_handle %d", req->handle);
  	    MPID_Abort(MPIR_Process.comm_world, mpi_errno, -1, NULL);
  	}
  #endif
  	/* FIXME: This makes request creation expensive.  We need to trim
  	   this to the basics, with additional setup for special-purpose 
  	   requests (think base class and inheritance).  For example, do we 
  	   *really* want to set the kind to UNDEFINED? And should the RMA 
  	   values be set only for RMA requests? */
  	MPIU_Object_set_ref(req, 1);
  	req->kind		   = MPID_REQUEST_UNDEFINED;
          MPID_cc_set(&req->cc, 1);
  	req->cc_ptr		   = &req->cc;
  	/* FIXME: status fields meaningful only for receive, and even then
  	   should not need to be set. */
  	req->status.MPI_SOURCE	   = MPI_UNDEFINED;
  	req->status.MPI_TAG	   = MPI_UNDEFINED;
  	req->status.MPI_ERROR	   = MPI_SUCCESS;
          MPIR_STATUS_SET_COUNT(req->status, 0);
          MPIR_STATUS_SET_CANCEL_BIT(req->status, FALSE);
  	req->comm		   = NULL;
          req->greq_fns              = NULL;
          req->errflag               = MPIR_ERR_NONE;
          req->request_completed_cb  = NULL;
  	req->dev.datatype_ptr	   = NULL;
  	req->dev.segment_ptr	   = NULL;
  	/* Masks and flags for channel device state in an MPID_Request */
  	req->dev.state		   = 0;
  	req->dev.cancel_pending	   = FALSE;
  	/* FIXME: RMA ops shouldn't need to be set except when creating a
  	   request for RMA operations */
  	req->dev.target_win_handle = MPI_WIN_NULL;
  	req->dev.source_win_handle = MPI_WIN_NULL;
          req->dev.target_lock_queue_entry = NULL;
  	req->dev.dataloop	   = NULL;
  	req->dev.iov_offset        = 0;
          req->dev.flags             = MPIDI_CH3_PKT_FLAG_NONE;
          req->dev.resp_request_handle = MPI_REQUEST_NULL;
          req->dev.user_buf          = NULL;
          req->dev.OnDataAvail       = NULL;
          req->dev.OnFinal           = NULL;
          req->dev.user_buf          = NULL;
          req->dev.drop_data         = FALSE;
          req->dev.tmpbuf            = NULL;
          req->dev.ext_hdr_ptr       = NULL;
          req->dev.ext_hdr_sz        = 0;
          req->dev.rma_target_ptr    = NULL;
          req->dev.request_handle    = MPI_REQUEST_NULL;
          req->dev.orig_segment_first = MPIDI_LOAD_RECV_IOV_ORIG_SEGMENT_FIRST_UNSET;
  #ifdef MPIDI_CH3_REQUEST_INIT
  	MPIDI_CH3_REQUEST_INIT(req);
  #endif
      }
      else
      {
  	/* FIXME: This fails to fail if debugging is turned off */
  	MPIU_DBG_MSG(CH3_CHANNEL,TYPICAL,"unable to allocate a request");
      }
      
      MPIDI_FUNC_EXIT(MPID_STATE_MPID_REQUEST_CREATE);
      return req;
  }
  
  
  /* ------------------------------------------------------------------------- */
  /* Here are the routines to manipulate the iovs in the requests              */
  /* ------------------------------------------------------------------------- */
  
  
  
  /*
   * MPIDI_CH3U_Request_load_send_iov()
   *
   * Fill the provided IOV with the next (or remaining) portion of data described
   * by the segment contained in the request structure.
   * If the density of IOV is not sufficient, pack the data into a send/receive 
   * buffer and point the IOV at the buffer.
   *
   * Expects sreq->dev.OnFinal to be initialized (even if it's NULL).
   */
  #undef FUNCNAME
  #define FUNCNAME MPIDI_CH3U_Request_load_send_iov
  #undef FCNAME
  #define FCNAME MPL_QUOTE(FUNCNAME)
  int MPIDI_CH3U_Request_load_send_iov(MPID_Request * const sreq, 
  				     MPL_IOV * const iov, int * const iov_n)
  {
      MPI_Aint last;
      int mpi_errno = MPI_SUCCESS;
      MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_REQUEST_LOAD_SEND_IOV);
  
      MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_REQUEST_LOAD_SEND_IOV);
      MPIU_Assert(sreq->dev.segment_ptr != NULL);
      last = sreq->dev.segment_size;
      MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,(MPIU_DBG_FDEST,
       "pre-pv: first=" MPIDI_MSG_SZ_FMT ", last=" MPIDI_MSG_SZ_FMT ", iov_n=%d",
  		      sreq->dev.segment_first, last, *iov_n));
      MPIU_Assert(sreq->dev.segment_first < last);
      MPIU_Assert(last > 0);
      MPIU_Assert(*iov_n > 0 && *iov_n <= MPL_IOV_LIMIT);
      MPID_Segment_pack_vector(sreq->dev.segment_ptr, sreq->dev.segment_first, 
  			     &last, iov, iov_n);
      MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,(MPIU_DBG_FDEST,
      "post-pv: first=" MPIDI_MSG_SZ_FMT ", last=" MPIDI_MSG_SZ_FMT ", iov_n=%d",
  		      sreq->dev.segment_first, last, *iov_n));
      MPIU_Assert(*iov_n > 0 && *iov_n <= MPL_IOV_LIMIT);
      
      if (last == sreq->dev.segment_size)
      {
  	MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,"remaining data loaded into IOV");
  	sreq->dev.OnDataAvail = sreq->dev.OnFinal;
      }
      else if ((last - sreq->dev.segment_first) / *iov_n >= MPIDI_IOV_DENSITY_MIN)
      {
  	MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,"more data loaded into IOV");
  	sreq->dev.segment_first = last;
  	sreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_SendReloadIOV;
      }
      else
      {
  	MPIDI_msg_sz_t data_sz;
  	int i, iov_data_copied;
  	
  	MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,"low density.  using SRBuf.");
  	    
  	data_sz = sreq->dev.segment_size - sreq->dev.segment_first;
  	if (!MPIDI_Request_get_srbuf_flag(sreq))
  	{
  	    MPIDI_CH3U_SRBuf_alloc(sreq, data_sz);
  	    /* --BEGIN ERROR HANDLING-- */
  	    if (sreq->dev.tmpbuf_sz == 0)
  	    {
  		MPIU_DBG_MSG(CH3_CHANNEL,TYPICAL,"SRBuf allocation failure");
  		mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, 
                                  FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 
  						 "**nomem %d", data_sz);
  		sreq->status.MPI_ERROR = mpi_errno;
  		goto fn_exit;
  	    }
  	    /* --END ERROR HANDLING-- */
  	}
  
  	iov_data_copied = 0;
  	for (i = 0; i < *iov_n; i++) {
  	    MPIU_Memcpy((char*) sreq->dev.tmpbuf + iov_data_copied, 
  		   iov[i].MPL_IOV_BUF, iov[i].MPL_IOV_LEN);
  	    iov_data_copied += iov[i].MPL_IOV_LEN;
  	}
  	sreq->dev.segment_first = last;
  
  	last = (data_sz <= sreq->dev.tmpbuf_sz - iov_data_copied) ? 
  	    sreq->dev.segment_size :
  	    sreq->dev.segment_first + sreq->dev.tmpbuf_sz - iov_data_copied;
  	MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,(MPIU_DBG_FDEST,
                 "pre-pack: first=" MPIDI_MSG_SZ_FMT ", last=" MPIDI_MSG_SZ_FMT,
  			  sreq->dev.segment_first, last));
  	MPID_Segment_pack(sreq->dev.segment_ptr, sreq->dev.segment_first, 
  			  &last, (char*) sreq->dev.tmpbuf + iov_data_copied);
  	MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,(MPIU_DBG_FDEST,
                "post-pack: first=" MPIDI_MSG_SZ_FMT ", last=" MPIDI_MSG_SZ_FMT,
  			   sreq->dev.segment_first, last));
  	iov[0].MPL_IOV_BUF = (MPL_IOV_BUF_CAST)sreq->dev.tmpbuf;
  	iov[0].MPL_IOV_LEN = last - sreq->dev.segment_first + iov_data_copied;
  	*iov_n = 1;
  	if (last == sreq->dev.segment_size)
  	{
  	    MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,"remaining data packed into SRBuf");
  	    sreq->dev.OnDataAvail = sreq->dev.OnFinal;
  	}
  	else 
  	{
  	    MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,"more data packed into SRBuf");
  	    sreq->dev.segment_first = last;
  	    sreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_SendReloadIOV;
  	}
      }
      
    fn_exit:
      MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_REQUEST_LOAD_SEND_IOV);
      return mpi_errno;
  }
  
  
  /*
   * MPIDI_CH3U_Request_load_recv_iov()
   *
   * Fill the request's IOV with the next (or remaining) portion of data 
   * described by the segment (also contained in the request
   * structure).  If the density of IOV is not sufficient, allocate a 
   * send/receive buffer and point the IOV at the buffer.
   */
  #undef FUNCNAME
  #define FUNCNAME MPIDI_CH3U_Request_load_recv_iov
  #undef FCNAME
  #define FCNAME MPL_QUOTE(FUNCNAME)
  int MPIDI_CH3U_Request_load_recv_iov(MPID_Request * const rreq)
  {
      MPI_Aint last;
      int mpi_errno = MPI_SUCCESS;
      MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_REQUEST_LOAD_RECV_IOV);
  
      MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_REQUEST_LOAD_RECV_IOV);
  
      if (rreq->dev.orig_segment_first == MPIDI_LOAD_RECV_IOV_ORIG_SEGMENT_FIRST_UNSET) {
          rreq->dev.orig_segment_first = rreq->dev.segment_first;
      }
  
      if (rreq->dev.segment_first < rreq->dev.segment_size)
      {
  	/* still reading data that needs to go into the user buffer */
  	
  	if (MPIDI_Request_get_type(rreq) != MPIDI_REQUEST_TYPE_ACCUM_RECV &&
              MPIDI_Request_get_type(rreq) != MPIDI_REQUEST_TYPE_GET_ACCUM_RECV &&
              MPIDI_Request_get_srbuf_flag(rreq))
  	{
  	    MPIDI_msg_sz_t data_sz;
  	    MPIDI_msg_sz_t tmpbuf_sz;
  
  	    /* Once a SRBuf is in use, we continue to use it since a small 
  	       amount of data may already be present at the beginning
  	       of the buffer.  This data is left over from the previous unpack,
  	       most like a result of alignment issues.  NOTE: we
  	       could force the use of the SRBuf only 
  	       when (rreq->dev.tmpbuf_off > 0)... */
  	    
  	    data_sz = rreq->dev.segment_size - rreq->dev.segment_first - 
  		rreq->dev.tmpbuf_off;
  	    MPIU_Assert(data_sz > 0);
  	    tmpbuf_sz = rreq->dev.tmpbuf_sz - rreq->dev.tmpbuf_off;
  	    if (data_sz > tmpbuf_sz)
  	    {
  		data_sz = tmpbuf_sz;
  	    }
  	    rreq->dev.iov[0].MPL_IOV_BUF = 
  		(MPL_IOV_BUF_CAST)((char *) rreq->dev.tmpbuf + 
  				    rreq->dev.tmpbuf_off);
  	    rreq->dev.iov[0].MPL_IOV_LEN = data_sz;
              rreq->dev.iov_offset = 0;
  	    rreq->dev.iov_count = 1;
  	    MPIU_Assert(rreq->dev.segment_first - rreq->dev.orig_segment_first + data_sz +
  			rreq->dev.tmpbuf_off <= rreq->dev.recv_data_sz);
  	    if (rreq->dev.segment_first - rreq->dev.orig_segment_first + data_sz + rreq->dev.tmpbuf_off ==
  		rreq->dev.recv_data_sz)
  	    {
  		MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
  		  "updating rreq to read the remaining data into the SRBuf");
  		rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_UnpackSRBufComplete;
                  rreq->dev.orig_segment_first = MPIDI_LOAD_RECV_IOV_ORIG_SEGMENT_FIRST_UNSET;
  	    }
  	    else
  	    {
  		MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
  		       "updating rreq to read more data into the SRBuf");
  		rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_UnpackSRBufReloadIOV;
  	    }
  	    goto fn_exit;
  	}
  	
  	last = rreq->dev.segment_size;
  	rreq->dev.iov_count = MPL_IOV_LIMIT;
  	rreq->dev.iov_offset = 0;
  	MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,(MPIU_DBG_FDEST,
     "pre-upv: first=" MPIDI_MSG_SZ_FMT ", last=" MPIDI_MSG_SZ_FMT ", iov_n=%d",
  			  rreq->dev.segment_first, last, rreq->dev.iov_count));
  	MPIU_Assert(rreq->dev.segment_first < last);
  	MPIU_Assert(last > 0);
  	MPID_Segment_unpack_vector(rreq->dev.segment_ptr, 
  				   rreq->dev.segment_first,
  				   &last, &rreq->dev.iov[0], &rreq->dev.iov_count);
  	MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,(MPIU_DBG_FDEST,
     "post-upv: first=" MPIDI_MSG_SZ_FMT ", last=" MPIDI_MSG_SZ_FMT ", iov_n=%d, iov_offset=%lld",
  			  rreq->dev.segment_first, last, rreq->dev.iov_count, (long long)rreq->dev.iov_offset));
  	MPIU_Assert(rreq->dev.iov_count >= 0 && rreq->dev.iov_count <= 
  		    MPL_IOV_LIMIT);
  
  	/* --BEGIN ERROR HANDLING-- */
  	if (rreq->dev.iov_count == 0)
  	{
  	    /* If the data can't be unpacked, the we have a mis-match between
  	       the datatype and the amount of data received.  Adjust
  	       the segment info so that the remaining data is received and 
  	       thrown away. */
  	    rreq->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS, 
  		       MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TYPE,
  		       "**dtypemismatch", 0);
              MPIR_STATUS_SET_COUNT(rreq->status, rreq->dev.segment_first);
  	    rreq->dev.segment_size = rreq->dev.segment_first;
  	    mpi_errno = MPIDI_CH3U_Request_load_recv_iov(rreq);
  	    goto fn_exit;
  	}
          else
          {
              MPIU_Assert(rreq->dev.iov_offset < rreq->dev.iov_count);
          }
  	/* --END ERROR HANDLING-- */
  
  	if (last == rreq->dev.recv_data_sz + rreq->dev.orig_segment_first)
  	{
  	    MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
       "updating rreq to read the remaining data directly into the user buffer");
  	    /* Eventually, use OnFinal for this instead */
  	    rreq->dev.OnDataAvail = rreq->dev.OnFinal;
              rreq->dev.orig_segment_first = MPIDI_LOAD_RECV_IOV_ORIG_SEGMENT_FIRST_UNSET;
  	}
  	else if (MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_ACCUM_RECV ||
                   MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_GET_ACCUM_RECV ||
                   (last == rreq->dev.segment_size ||
                    (last - rreq->dev.segment_first) / rreq->dev.iov_count >= MPIDI_IOV_DENSITY_MIN))
  	{
  	    MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
  	     "updating rreq to read more data directly into the user buffer");
  	    rreq->dev.segment_first = last;
  	    rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_ReloadIOV;
  	}
  	else
  	{
  	    /* Too little data would have been received using an IOV.  
  	       We will start receiving data into a SRBuf and unpacking it
  	       later. */
  	    MPIU_Assert(MPIDI_Request_get_srbuf_flag(rreq) == FALSE);
  	    
  	    MPIDI_CH3U_SRBuf_alloc(rreq, 
  			    rreq->dev.segment_size - rreq->dev.segment_first);
  	    rreq->dev.tmpbuf_off = 0;
  	    /* --BEGIN ERROR HANDLING-- */
  	    if (rreq->dev.tmpbuf_sz == 0)
  	    {
  		/* FIXME - we should drain the data off the pipe here, but we 
  		   don't have a buffer to drain it into.  should this be
  		   a fatal error? */
  		MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,"SRBuf allocation failure");
  		mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, 
  			      FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 
  			 "**nomem %d", 
  			 rreq->dev.segment_size - rreq->dev.segment_first);
  		rreq->status.MPI_ERROR = mpi_errno;
  		goto fn_exit;
  	    }
  	    /* --END ERROR HANDLING-- */
  
  	    /* fill in the IOV using a recursive call */
  	    mpi_errno = MPIDI_CH3U_Request_load_recv_iov(rreq);
  	}
      }
      else
      {
  	/* receive and toss any extra data that does not fit in the user's 
  	   buffer */
  	MPIDI_msg_sz_t data_sz;
  
  	data_sz = rreq->dev.recv_data_sz - rreq->dev.segment_first;
  	if (!MPIDI_Request_get_srbuf_flag(rreq))
  	{
  	    MPIDI_CH3U_SRBuf_alloc(rreq, data_sz);
  	    /* --BEGIN ERROR HANDLING-- */
  	    if (rreq->dev.tmpbuf_sz == 0)
  	    {
  		MPIU_DBG_MSG(CH3_CHANNEL,TYPICAL,"SRBuf allocation failure");
  		mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL, 
  			       FCNAME, __LINE__, MPI_ERR_OTHER, "**nomem", 0);
  		rreq->status.MPI_ERROR = mpi_errno;
  		goto fn_exit;
  	    }
  	    /* --END ERROR HANDLING-- */
  	}
  
  	if (data_sz <= rreq->dev.tmpbuf_sz)
  	{
  	    MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
  	    "updating rreq to read overflow data into the SRBuf and complete");
  	    rreq->dev.iov[0].MPL_IOV_LEN = data_sz;
  	    MPIU_Assert(MPIDI_Request_get_type(rreq) == MPIDI_REQUEST_TYPE_RECV);
  	    /* Eventually, use OnFinal for this instead */
  	    rreq->dev.OnDataAvail = rreq->dev.OnFinal;
              rreq->dev.orig_segment_first = MPIDI_LOAD_RECV_IOV_ORIG_SEGMENT_FIRST_UNSET;
  	}
  	else
  	{
  	    MPIU_DBG_MSG(CH3_CHANNEL,VERBOSE,
  	  "updating rreq to read overflow data into the SRBuf and reload IOV");
  	    rreq->dev.iov[0].MPL_IOV_LEN = rreq->dev.tmpbuf_sz;
  	    rreq->dev.segment_first += rreq->dev.tmpbuf_sz;
  	    rreq->dev.OnDataAvail = MPIDI_CH3_ReqHandler_ReloadIOV;
  	}
  	
  	rreq->dev.iov[0].MPL_IOV_BUF = (MPL_IOV_BUF_CAST)rreq->dev.tmpbuf;
  	rreq->dev.iov_count = 1;
      }
      
    fn_exit:
      MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_REQUEST_LOAD_RECV_IOV);
      return mpi_errno;
  }
  
  /*
   * MPIDI_CH3U_Request_unpack_srbuf
   *
   * Unpack data from a send/receive buffer into the user buffer.
   */
  #undef FUNCNAME
  #define FUNCNAME MPIDI_CH3U_Request_unpack_srbuf
  #undef FCNAME
  #define FCNAME MPL_QUOTE(FUNCNAME)
  int MPIDI_CH3U_Request_unpack_srbuf(MPID_Request * rreq)
  {
      MPI_Aint last;
      int tmpbuf_last;
      int mpi_errno = MPI_SUCCESS;
      MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_REQUEST_UNPACK_SRBUF);
      
      MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_REQUEST_UNPACK_SRBUF);
  
      tmpbuf_last = (int)(rreq->dev.segment_first + rreq->dev.tmpbuf_sz);
      if (rreq->dev.segment_size < tmpbuf_last)
      {
  	tmpbuf_last = (int)rreq->dev.segment_size;
      }
      last = tmpbuf_last;
      MPID_Segment_unpack(rreq->dev.segment_ptr, rreq->dev.segment_first, 
  			&last, rreq->dev.tmpbuf);
      if (last == 0 || last == rreq->dev.segment_first)
      {
  	/* --BEGIN ERROR HANDLING-- */
  	/* If no data can be unpacked, then we have a datatype processing 
  	   problem.  Adjust the segment info so that the remaining
  	   data is received and thrown away. */
  	MPIR_STATUS_SET_COUNT(rreq->status, rreq->dev.segment_first);
  	rreq->dev.segment_size = rreq->dev.segment_first;
  	rreq->dev.segment_first += tmpbuf_last;
  	rreq->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS, 
  		       MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TYPE,
  		       "**dtypemismatch", 0);
  	/* --END ERROR HANDLING-- */
      }
      else if (tmpbuf_last == rreq->dev.segment_size)
      {
  	/* --BEGIN ERROR HANDLING-- */
  	if (last != tmpbuf_last)
  	{
  	    /* received data was not entirely consumed by unpack() because too
  	       few bytes remained to fill the next basic datatype.
  	       Note: the segment_first field is set to segment_last so that if
  	       this is a truncated message, extra data will be read
  	       off the pipe. */
  	    MPIR_STATUS_SET_COUNT(rreq->status, last);
  	    rreq->dev.segment_size = last;
  	    rreq->dev.segment_first = tmpbuf_last;
  	    rreq->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS, 
  		  MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TYPE,
  							  "**dtypemismatch", 0);
  	}
  	/* --END ERROR HANDLING-- */
      }
      else
      {
  	rreq->dev.tmpbuf_off = (int)(tmpbuf_last - last);
  	if (rreq->dev.tmpbuf_off > 0)
  	{
  	    /* move any remaining data to the beginning of the buffer.  
  	       Note: memmove() is used since the data regions could
                 overlap. */
  	    memmove(rreq->dev.tmpbuf, (char *) rreq->dev.tmpbuf + 
  		    (last - rreq->dev.segment_first), rreq->dev.tmpbuf_off);
  	}
  	rreq->dev.segment_first = last;
      }
  
      MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_REQUEST_UNPACK_SRBUF);
      return mpi_errno;
  }
  
  /*
   * MPIDI_CH3U_Request_unpack_uebuf
   *
   * Copy/unpack data from an "unexpected eager buffer" into the user buffer.
   */
  #undef FUNCNAME
  #define FUNCNAME MPIDI_CH3U_Request_unpack_uebuf
  #undef FCNAME
  #define FCNAME MPL_QUOTE(FUNCNAME)
  int MPIDI_CH3U_Request_unpack_uebuf(MPID_Request * rreq)
  {
      int dt_contig;
      MPI_Aint dt_true_lb;
      MPIDI_msg_sz_t userbuf_sz;
      MPID_Datatype * dt_ptr;
      MPIDI_msg_sz_t unpack_sz;
      int mpi_errno = MPI_SUCCESS;
      MPIDI_STATE_DECL(MPID_STATE_MPIDI_CH3U_REQUEST_UNPACK_UEBUF);
      MPIDI_STATE_DECL(MPID_STATE_MEMCPY);
  
      MPIDI_FUNC_ENTER(MPID_STATE_MPIDI_CH3U_REQUEST_UNPACK_UEBUF);
  
      MPIDI_Datatype_get_info(rreq->dev.user_count, rreq->dev.datatype, 
  			    dt_contig, userbuf_sz, dt_ptr, dt_true_lb);
      
      if (rreq->dev.recv_data_sz <= userbuf_sz)
      {
  	unpack_sz = rreq->dev.recv_data_sz;
      }
      else
      {
  	/* --BEGIN ERROR HANDLING-- */
  	MPIU_DBG_MSG_FMT(CH3_CHANNEL,VERBOSE,(MPIU_DBG_FDEST,
        "receive buffer overflow; message truncated, msg_sz=" MPIDI_MSG_SZ_FMT 
  	      ", buf_sz=" MPIDI_MSG_SZ_FMT, 
                  rreq->dev.recv_data_sz, userbuf_sz));
  	unpack_sz = userbuf_sz;
  	MPIR_STATUS_SET_COUNT(rreq->status, userbuf_sz);
  	rreq->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS, 
  		 MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TRUNCATE,
  		 "**truncate", "**truncate %d %d", 
                   rreq->dev.recv_data_sz, userbuf_sz);
  	/* --END ERROR HANDLING-- */
      }
  
      if (unpack_sz > 0)
      {
  	if (dt_contig)
  	{
  	    /* TODO - check that amount of data is consistent with datatype.  
  	       In other words, if we were to use Segment_unpack()
  	       would last = unpack?  If not we should return an error 
  	       (unless configured with --enable-fast) */
  	    MPIDI_FUNC_ENTER(MPID_STATE_MEMCPY);
  	    MPIU_Memcpy((char *)rreq->dev.user_buf + dt_true_lb, rreq->dev.tmpbuf,
  		   unpack_sz);
  	    MPIDI_FUNC_EXIT(MPID_STATE_MEMCPY);
  	}
  	else
  	{
  	    MPID_Segment seg;
  	    MPI_Aint last;
  
  	    MPID_Segment_init(rreq->dev.user_buf, rreq->dev.user_count, 
  			      rreq->dev.datatype, &seg, 0);
  	    last = unpack_sz;
  	    MPID_Segment_unpack(&seg, 0, &last, rreq->dev.tmpbuf);
  	    if (last != unpack_sz)
  	    {
  		/* --BEGIN ERROR HANDLING-- */
  		/* received data was not entirely consumed by unpack() 
  		   because too few bytes remained to fill the next basic
  		   datatype */
  		MPIR_STATUS_SET_COUNT(rreq->status, last);
  		rreq->status.MPI_ERROR = MPIR_Err_create_code(MPI_SUCCESS, 
                           MPIR_ERR_RECOVERABLE, FCNAME, __LINE__, MPI_ERR_TYPE,
  			 "**dtypemismatch", 0);
  		/* --END ERROR HANDLING-- */
  	    }
  	}
      }
  
      MPIDI_FUNC_EXIT(MPID_STATE_MPIDI_CH3U_REQUEST_UNPACK_UEBUF);
      return mpi_errno;
  }
  
  int MPID_Request_complete(MPID_Request *req)
  {
      int incomplete;
      int mpi_errno = MPI_SUCCESS;
      static int called_cnt = 0;
  
      MPIU_Assert(called_cnt <= REQUEST_CB_DEPTH);
      called_cnt++;
  
      MPIDI_CH3U_Request_decrement_cc(req, &incomplete);
      if (!incomplete) {
          /* trigger request_completed callback function */
          if (req->request_completed_cb != NULL) {
              mpi_errno = req->request_completed_cb(req);
              if (mpi_errno != MPI_SUCCESS) {
                  MPIR_ERR_POP(mpi_errno);
              }
          }
  
  	MPID_Request_release(req);
  	MPIDI_CH3_Progress_signal_completion();
      }
  
   fn_exit:
      called_cnt--;
      return mpi_errno;
   fn_fail:
      goto fn_exit;
  }
  
  void MPID_Request_release(MPID_Request *req)
  {
      int inuse;
  
      MPIR_Request_release_ref(req, &inuse);
      if (inuse == 0) {
          MPIU_DBG_MSG_P(CH3_CHANNEL,VERBOSE,
                         "freeing request, handle=0x%08x", req->handle);
  
  #ifdef MPICH_DBG_OUTPUT
          /*MPIU_Assert(HANDLE_GET_MPI_KIND(req->handle) == MPID_REQUEST);*/
          if (HANDLE_GET_MPI_KIND(req->handle) != MPID_REQUEST)
          {
              int mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
                                                   FCNAME, __LINE__, MPI_ERR_OTHER,
                                                   "**invalid_handle", "**invalid_handle %d", req->handle);
              MPID_Abort(MPIR_Process.comm_world, mpi_errno, -1, NULL);
          }
          /* XXX DJG FIXME should we be checking this? */
          /*MPIU_Assert(req->ref_count == 0);*/
          if (req->ref_count != 0)
          {
              int mpi_errno = MPIR_Err_create_code(MPI_SUCCESS, MPIR_ERR_FATAL,
                                                   FCNAME, __LINE__, MPI_ERR_OTHER,
                                                   "**invalid_refcount", "**invalid_refcount %d", req->ref_count);
              MPID_Abort(MPIR_Process.comm_world, mpi_errno, -1, NULL);
          }
  #endif
  
          /* FIXME: We need a better way to handle these so that we do
             not always need to initialize these fields and check them
             when we destroy a request */
          /* FIXME: We need a way to call these routines ONLY when the
             related ref count has become zero. */
          if (req->comm != NULL) {
              MPIR_Comm_release(req->comm);
          }
  
          if (req->greq_fns != NULL) {
              MPIU_Free(req->greq_fns);
          }
  
          if (req->dev.datatype_ptr != NULL) {
              MPID_Datatype_release(req->dev.datatype_ptr);
          }
  
          if (req->dev.segment_ptr != NULL) {
              MPID_Segment_free(req->dev.segment_ptr);
          }
  
          if (MPIDI_Request_get_srbuf_flag(req)) {
              MPIDI_CH3U_SRBuf_free(req);
          }
  
          if (req->dev.ext_hdr_ptr != NULL) {
              MPIU_Free(req->dev.ext_hdr_ptr);
          }
  
          MPIU_Handle_obj_free(&MPID_Request_mem, req);
      }
  }
  