text: /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
  /* vim: set ts=8 sts=4 sw=4 noexpandtab : */
  /*
   *
   */
  
  
  
  #include "mpid_nem_impl.h"
  #include "llc_impl.h"
  
  //#define MPID_NEM_LLC_DEBUG_POLL
  #ifdef MPID_NEM_LLC_DEBUG_POLL
  #define dprintf printf
  #else
  #define dprintf(...)
  #endif
  
  /* function prototypes */
  
  static void MPID_nem_llc_send_handler(void *cba, uint64_t * p_reqid);
  static void MPID_nem_llc_recv_handler(void *vp_vc, uint64_t raddr, void *buf, size_t bsz);
  
  #undef FUNCNAME
  #define FUNCNAME MPID_nem_llc_poll
  #undef FCNAME
  #define FCNAME MPIDI_QUOTE(FUNCNAME)
  int MPID_nem_llc_poll(int in_blocking_progress)
  {
      int mpi_errno = MPI_SUCCESS;
      MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LLC_POLL);
  
      MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LLC_POLL);
  
      {
          int rc;
          rc = llc_poll(in_blocking_progress, MPID_nem_llc_send_handler, MPID_nem_llc_recv_handler);
          if (rc != 0) {
              mpi_errno = MPI_ERR_OTHER;
              MPIU_ERR_POP(mpi_errno);
          }
      }
  
    fn_exit:
      MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LLC_POLL);
      return mpi_errno;
    fn_fail:
      goto fn_exit;
  }
  
  #undef FUNCNAME
  #define FUNCNAME MPID_nem_llc_send_handler
  #undef FCNAME
  #define FCNAME MPIDI_QUOTE(FUNCNAME)
  static void MPID_nem_llc_send_handler(void *cba, uint64_t * p_reqid)
  {
      /* int mpi_errno = 0; */
      MPID_Request *sreq = cba;   /* from llc_writev(,,,,cbarg,) */
      MPID_Request_kind_t kind;
      /* MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LLC_SEND_HANDLER); */
  
      /* MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LLC_SEND_HANDLER); */
  
      MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "llc_send_handler");
  
      MPIU_Assert(sreq != NULL);
  
      if (sreq == (void *) 0xdeadbeefUL) {
          MPIDI_VC_t *vc = (void *) p_reqid[0];
          MPID_nem_llc_vc_area *vc_llc;
  
          MPIU_Assert(vc != NULL);
          /* printf("from credit %p (pg_rank %d)\n", vc, vc->pg_rank); */
  
          vc_llc = VC_LLC(vc);
          MPID_nem_llc_send_queued(vc, &vc_llc->send_queue);
  
          p_reqid[0] = !MPIDI_CH3I_Sendq_empty(vc_llc->send_queue);
          MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "llc_send_handler");
          MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "send queue %d", (unsigned int) p_reqid[0]);
  
          goto fn_exit;
      }
  
      kind = sreq->kind;
      switch (kind) {
          unsigned int reqtype;
      case MPID_REQUEST_SEND:
      case MPID_PREQUEST_SEND:{
              reqtype = MPIDI_Request_get_type(sreq);
  
              /* Free temporal buffer for non-contiguous data.
               * MPIDI_Request_create_sreq (in mpid_isend.c) sets req->dev.datatype.
               * A control message has a req_type of MPIDI_REQUEST_TYPE_RECV and
               * msg_type of MPIDI_REQUEST_EAGER_MSG because
               * control message send follows
               * MPIDI_CH3_iStartMsg/v-->MPID_nem_llc_iStartContigMsg-->MPID_nem_llc_iSendContig
               * and MPID_nem_llc_iSendContig set req->dev.state to zero
               * because MPID_Request_create (in src/mpid/ch3/src/ch3u_request.c)
               * sets it to zero. In addition, eager-short message has req->comm of zero. */
  #ifndef	notdef_leak_0001_hack
              /* See also MPIDI_CH3_Request_create and _destory() */
              /*     in src/mpid/ch3/src/ch3u_request.c */
  #endif /* notdef_leak_0001_hack */
              if (reqtype != MPIDI_REQUEST_TYPE_RECV && sreq->comm) {
                  /* Exclude control messages which have MPIDI_REQUEST_TYPE_RECV.
                   * Note that RMA messages should be included.
                   * Exclude eager-short by requiring req->comm != 0. */
                  int is_contig;
                  MPID_Datatype_is_contig(sreq->dev.datatype, &is_contig);
                  if (!is_contig && REQ_FIELD(sreq, pack_buf)) {
                      dprintf("llc_send_handler,non-contiguous,free pack_buf\n");
                      MPIU_Free(REQ_FIELD(sreq, pack_buf));
                  }
              }
  
              if ((REQ_FIELD(sreq, rma_buf) != NULL && sreq->dev.datatype_ptr &&
                   sreq->dev.segment_size > 0)) {
                  MPIU_Free(REQ_FIELD(sreq, rma_buf));    // allocated in MPID_nem_llc_SendNoncontig
                  REQ_FIELD(sreq, rma_buf) = NULL;
              }
  
              /* sreq: src/mpid/ch3/include/mpidpre.h */
              {
                  MPIDI_VC_t *vc;
                  int (*reqFn) (MPIDI_VC_t * vc, MPID_Request * sreq, int *complete);
                  int complete;
                  int r_mpi_errno;
  
                  p_reqid[0] = 0 /* REQ_LLC(sreq)->woff */ ;
  
                  vc = sreq->ch.vc;       /* before callback */
                  reqFn = sreq->dev.OnDataAvail;
                  if (reqFn == 0) {
                      MPIU_Assert(reqtype != MPIDI_REQUEST_TYPE_GET_RESP);
  
                      MPIDI_CH3U_Request_complete(sreq);
                      MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete");
                  }
                  else {
                      complete = 0;
                      r_mpi_errno = reqFn(vc, sreq, &complete);
                      if (r_mpi_errno)
                          MPIU_ERR_POP(r_mpi_errno);
                      if (complete == 0) {
                          MPIU_Assert(complete == TRUE);
                      }
                      MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, ".... complete2");
                  }
  
                  /* push queued messages */
                  {
                      MPID_nem_llc_vc_area *vc_llc = VC_LLC(vc);
  
                      MPID_nem_llc_send_queued(vc, &vc_llc->send_queue);
                  }
              }
              break;
          }
      default:
          printf("send_handler,unknown kind=%08x\n", sreq->kind);
          MPID_nem_llc_segv;
          break;
      }
  
    fn_exit:
      /* MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LLC_SEND_HANDLER); */
      return /* mpi_errno */ ;
    fn_fail:
      goto fn_exit;
  }
  
  #undef FUNCNAME
  #define FUNCNAME MPID_nem_llc_recv_handler
  #undef FCNAME
  #define FCNAME MPIDI_QUOTE(FUNCNAME)
  static void MPID_nem_llc_recv_handler(void *vp_vc, uint64_t raddr, void *buf, size_t bsz)
  {
      int mpi_errno = 0;
      MPIDI_VC_t *vc = vp_vc;
      /* MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LLC_RECV_HANDLER); */
  
      /* MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LLC_RECV_HANDLER); */
  
      MPIU_DBG_MSG(CH3_CHANNEL, VERBOSE, "llc_recv_handler");
  
      {
          int pg_rank = (int) raddr;
          MPIDI_PG_t *pg = MPIDI_Process.my_pg;
          MPIDI_VC_t *vc_from_pg = 0;
  
          if ((pg != 0)
              && ((pg_rank >= 0) && (pg_rank < MPIDI_PG_Get_size(pg)))
  ) {
              /*
               * MPIDI_Comm_get_vc_set_active(comm, rank, &vc);
               */
              MPIDI_PG_Get_vc_set_active(pg, pg_rank, &vc_from_pg);
          }
          else {
              MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "bad vc %p or", pg);
              MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "bad pg_rank %d", pg_rank);
              MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "bad pg_rank < %d", MPIDI_PG_Get_size(pg));
              vc_from_pg = vc;    /* XXX */
          }
          if (vc != vc_from_pg) {
              MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "bad vc for pg_rank %d", pg_rank);
          }
          if (vc == 0) {
              vc = vc_from_pg;
          }
      }
      if (vc != 0) {
          mpi_errno = MPID_nem_handle_pkt(vc, buf, bsz);
          if (mpi_errno != 0) {
              MPIU_DBG_MSG_D(CH3_CHANNEL, VERBOSE, "MPID_nem_handle_pkt() = %d", mpi_errno);
          }
      }
  
    fn_exit:
      /* MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LLC_RECV_HANDLER); */
      return;
      //fn_fail:
      goto fn_exit;
  }
  
  #undef FUNCNAME
  #define FUNCNAME MPID_nem_llc_recv_posted
  #undef FCNAME
  #define FCNAME MPIDI_QUOTE(FUNCNAME)
  int MPID_nem_llc_recv_posted(struct MPIDI_VC *vc, struct MPID_Request *req)
  {
      int mpi_errno = MPI_SUCCESS, llc_errno;
      int dt_contig;
      MPIDI_msg_sz_t data_sz;
      MPID_Datatype *dt_ptr;
      MPI_Aint dt_true_lb;
      int i;
  
      MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LLC_RECV_POSTED);
      MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LLC_RECV_POSTED);
  
      /* req->dev.datatype is set in MPID_irecv --> MPIDI_CH3U_Recvq_FDU_or_AEP */
      MPIDI_Datatype_get_info(req->dev.user_count, req->dev.datatype, dt_contig, data_sz, dt_ptr,
                              dt_true_lb);
  
      /* Don't save VC because it's not used in llc_poll */
  
      /* Save data size for llc_poll */
      req->dev.recv_data_sz = data_sz;
  
  #if 0   /* FIXME : vc is NULL when rank is MPI_ANY_SOURCE */
      dprintf
          ("llc_recv_posted,%d<-%d,vc=%p,req=%p,user_buf=%p,data_sz=%ld,datatype=%08x,dt_contig=%d\n",
           MPIDI_Process.my_pg_rank, vc->pg_rank, vc, req, req->dev.user_buf, req->dev.recv_data_sz,
           req->dev.datatype, dt_contig);
  #endif
  
      void *write_to_buf;
      if (dt_contig) {
          write_to_buf = (void *) ((char *) req->dev.user_buf + dt_true_lb);
      }
      else {
          REQ_FIELD(req, pack_buf) = MPIU_Malloc(data_sz);
          MPIU_ERR_CHKANDJUMP(!REQ_FIELD(req, pack_buf), mpi_errno, MPI_ERR_OTHER, "**outofmemory");
          write_to_buf = REQ_FIELD(req, pack_buf);
      }
  
      int LLC_my_rank;
      LLC_comm_rank(LLC_COMM_MPICH, &LLC_my_rank);
      dprintf("llc_isend,LLC_my_rank=%d\n", LLC_my_rank);
  
  #if 0   /* FIXME : vc is NULL when rank is MPI_ANY_SOURCE */
      dprintf("llc_recv_posted,remote_endpoint_addr=%ld\n", VC_FIELD(vc, remote_endpoint_addr));
  #endif
  
      LLC_cmd_t *cmd = LLC_cmd_alloc2(1, 1, 1);
  
      cmd[0].opcode = LLC_OPCODE_RECV;
      cmd[0].comm = LLC_COMM_MPICH;
      cmd[0].req_id = (uint64_t) cmd;
  
      if (req->dev.match.parts.rank == MPI_ANY_SOURCE) {
          cmd[0].rank = LLC_ANY_SOURCE;
          cmd[0].mask.rank = 0;
      }
      else {
          cmd[0].rank = VC_FIELD(vc, remote_endpoint_addr);
      }
  
      /* req->comm is set in MPID_irecv --> MPIDI_CH3U_Recvq_FDU_or_AEP */
      if (req->dev.match.parts.tag == MPI_ANY_TAG) {
          *(int32_t *) ((uint8_t *) & cmd[0].tag) = LLC_ANY_TAG;
          *(int32_t *) ((uint8_t *) & cmd[0].mask.tag) = 0;
      }
      else {
          *(int32_t *) ((uint8_t *) & cmd[0].tag) = req->dev.match.parts.tag;
      }
  
      *(MPIR_Context_id_t *) ((uint8_t *) & cmd[0].tag + sizeof(int32_t)) =
          req->dev.match.parts.context_id;
      MPIU_Assert(sizeof(LLC_tag_t) >= sizeof(int32_t) + sizeof(MPIR_Context_id_t));
      memset((uint8_t *) & cmd[0].tag + sizeof(int32_t) + sizeof(MPIR_Context_id_t),
             0, sizeof(LLC_tag_t) - sizeof(int32_t) - sizeof(MPIR_Context_id_t));
  
  
      dprintf("llc_recv_posted,tag=");
      for (i = 0; i < sizeof(LLC_tag_t); i++) {
          dprintf("%02x", (int) *((uint8_t *) & cmd[0].tag + i));
      }
      dprintf("\n");
  
  
      cmd[0].iov_local[0].addr = (uint64_t) write_to_buf;
      cmd[0].iov_local[0].length = data_sz;
      cmd[0].niov_local = 1;
  
      cmd[0].iov_remote[0].addr = 0;
      cmd[0].iov_remote[0].length = data_sz;;
      cmd[0].niov_remote = 1;
  
      ((struct llc_cmd_area *) cmd[0].usr_area)->cbarg = req;
      if (req->dev.match.parts.rank == MPI_ANY_SOURCE) {
          ((struct llc_cmd_area *) cmd[0].usr_area)->raddr = MPI_ANY_SOURCE;      /* FIXME : should 0 ? */
      }
      else {
          ((struct llc_cmd_area *) cmd[0].usr_area)->raddr = VC_FIELD(vc, remote_endpoint_addr);
      }
  
      REQ_FIELD(req, cmds) = cmd;
  
      llc_errno = LLC_post(cmd, 1);
      MPIU_ERR_CHKANDJUMP(llc_errno != LLC_SUCCESS, mpi_errno, MPI_ERR_OTHER, "**LLC_post");
  
    fn_exit:
      MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LLC_RECV_POSTED);
      return mpi_errno;
    fn_fail:
      goto fn_exit;
  }
  
  #undef FUNCNAME
  #define FUNCNAME MPID_nem_llc_anysource_posted
  #undef FCNAME
  #define FCNAME MPIDI_QUOTE(FUNCNAME)
  void MPID_nem_llc_anysource_posted(MPID_Request * req)
  {
      int mpi_errno = MPI_SUCCESS;
      MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LLC_AYSOURCE_POSTED);
      MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LLC_AYSOURCE_POSTED);
  
      mpi_errno = MPID_nem_llc_recv_posted(NULL, req);
      MPIU_Assert(mpi_errno == MPI_SUCCESS);
  
      MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LLC_AYSOURCE_POSTED);
  }
  
  #undef FUNCNAME
  #define FUNCNAME MPID_nem_llc_anysource_matched
  #undef FCNAME
  #define FCNAME MPIDI_QUOTE(FUNCNAME)
  int MPID_nem_llc_anysource_matched(MPID_Request * req)
  {
      int matched = FALSE;
  
      MPIDI_STATE_DECL(MPID_STATE_MPID_NEM_LLC_ANYSOURCE_MATCHED);
      MPIDI_FUNC_ENTER(MPID_STATE_MPID_NEM_LLC_ANYSOURCE_MATCHED);
  
      /* FIXME : How to call a cancel_recv function */
      /* If LLC_postedq is still having this request, delete it.
       * Ohterwise, return TURE */
      matched = LLC_req_approve_recv((LLC_cmd_t *) REQ_FIELD(req, cmds));
  
      MPIDI_FUNC_EXIT(MPID_STATE_MPID_NEM_LLC_ANYSOURCE_MATCHED);
  
      return matched;
  }
  