dart/gni/ds_base_gni.c
static int sp_register_completion(struct rpc_server *rpc_s, struct msg_buf *msg)
{
	struct dart_server *ds = ds_ref_from_rpc(rpc_s);
	struct app_info *app;
	struct node_id *peer;
	int err = -ENOMEM;
	int i, j;

	struct ptlid_map *dcreg;
	struct smsg_attr_reg *sar;
	struct gni_smsg_attr_info *cur_attr_info;
	gni_smsg_attr_t	*smsg_attr;
	
	gni_return_t status;
	void *recv_buffer;

	int info_size = msg->size;
	int num_cp = info_size/(sizeof(struct ptlid_map) + sizeof(gni_smsg_attr_t));	

	recv_buffer = msg->msg_data;
	dcreg = (struct ptlid_map *)recv_buffer;
	smsg_attr = (gni_smsg_attr_t *)(dcreg+num_cp);

	cur_attr_info = ds->rpc_s->attr_info_start;
	while(1) {
		if(!cur_attr_info->next) {
			cur_attr_info->next = (struct gni_smsg_attr_info *)malloc(sizeof(struct gni_smsg_attr_info));
			memset(cur_attr_info->next, 0, sizeof(struct gni_smsg_attr_info));
			cur_attr_info = cur_attr_info->next;
			cur_attr_info->next = NULL;
			cur_attr_info->appid = dcreg->appid;
			break;
		}
		else {
			cur_attr_info = cur_attr_info->next;	
	    }
    }

	err = rpc_smsg_init(ds->rpc_s, cur_attr_info, num_cp);
	if(err != 0) {
		printf("Rank %d: failed for rpc_smsg_init. (%d)\n", ds->rpc_s->ptlmap.id, err);
		goto err_out;
	}

	peer = &ds->peer_tab[ds->size_sp-1];
	while(1) {
		if(!peer->next) {
			peer->next = (struct node_id *)malloc(num_cp * sizeof(struct node_id));
			memset(peer->next, 0, num_cp * sizeof(struct node_id));
			peer = peer->next;
			break;
		} else {
			peer = (struct node_id *)(peer->next + peer->next->peer_num - 1);
        }        
	}	

	//set up new peer_tab block for this app
	for(j = 0; j < num_cp; j++) {
		peer->ptlmap.nid = dcreg->nid;
		peer->ptlmap.pid = dcreg->pid;
		peer->ptlmap.appid = dcreg->appid;
		peer->ptlmap.id = dcreg->id;		
		peer->peer_rank = j;
		peer->peer_num = num_cp;

		peer->f_reg = 1;

		INIT_LIST_HEAD(&peer->req_list);
		peer->num_msg_at_peer = ds->rpc_s->max_num_msg;

		peer->num_msg_recv = 0;
		peer->num_msg_ret = 0;	  

		peer->sys_msg_recv = 0;
		peer->sys_msg_at_peer = ds->rpc_s->max_num_msg;
		peer->sys_msg_ret = 0;	 
		peer->next = NULL;


		status = GNI_EpCreate(ds->rpc_s->nic_hndl, ds->rpc_s->src_cq_hndl, &peer->ep_hndl);
		if (status != GNI_RC_SUCCESS)
		{
			printf("Fail: GNI_EpCreate returned error. %d.\n", status);
			goto err_free;
		}

		status = GNI_EpBind(peer->ep_hndl, peer->ptlmap.nid, peer->ptlmap.id);
		if (status != GNI_RC_SUCCESS)
		{
			printf("Fail: GNI_EpBind returned error. %d.\n", status);
			goto err_free;
		}

		status = GNI_EpCreate(ds->rpc_s->nic_hndl, ds->rpc_s->sys_cq_hndl, &peer->sys_ep_hndl);
		if (status != GNI_RC_SUCCESS)
		{
			printf("Rank %d: Fail GNI_EpCreate SYS returned error. %d.\n", ds->rpc_s->ptlmap.id, status);
			goto err_out;
		}
		status = GNI_EpBind(peer->sys_ep_hndl, peer->ptlmap.nid, peer->ptlmap.id);
		if (status != GNI_RC_SUCCESS)
		{
		printf("Rank %d: Fail GNI_EpBind SYS returned error. %d.\n", ds->rpc_s->ptlmap.id, status);
		goto err_out;
		}

		peer->remote_smsg_attr = *smsg_attr;	

		err = rpc_smsg_config(ds->rpc_s, cur_attr_info, peer);
		if (err != 0){
			printf("Rank %d: failed for config SMSG for peer %d. (%d)\n", ds->rpc_s->ptlmap.id, peer->ptlmap.id, err);
			goto err_out;
		}	

		dcreg++;
		peer++;
		smsg_attr++;

	}


	dcreg = (struct ptlid_map *)recv_buffer;
	app = app_find(ds, dcreg->appid);
	if (!app) {
		app = app_alloc();
		if (!app) 
			goto err_out;

		ds->num_cp += num_cp;

		app->app_id = dcreg->appid;
		app->app_num_peers = app->app_cnt_peers = num_cp;

		app->app_peer_tab = ds_get_peer(ds, dcreg->id);
		list_add(&app->app_entry, &ds->app_list);

		app->f_reg = 1;
	}

    err = ds_register_cp(ds, app);
	if (err != 0){
		printf("Rank %d: failed for ds_register_cp for application %d. (%d)\n", ds->rpc_s->ptlmap.id, dcreg->appid, err);
		goto err_out;
	}

	free(msg->msg_data);
	free(msg);

    return 0;

err_free:
	printf("'%s()': failed with %d.\n", __func__, status);
	return status;

 err_out:
	printf("(%s): failed. (%d)\n", __func__, err);
	return err;
}
// slave server gets app info and finish local reg for this app

slave server gets app info and finish local reg for this app
slave server gets app info and finish local reg for this app
static int sp_register_completion(struct rpc_server *rpc_s, struct msg_buf *msg)
{
	struct dart_server *ds = ds_ref_from_rpc(rpc_s);
	struct app_info *app;
	struct node_id *peer;
	int err = -ENOMEM;
	int i, j;
	struct ptlid_map *dcreg;
	struct smsg_attr_reg *sar;
	struct gni_smsg_attr_info *cur_attr_info;
	gni_smsg_attr_t	*smsg_attr;
	gni_return_t status;
	void *recv_buffer;
	int info_size = msg->size;
	int num_cp = info_size/(sizeof(struct ptlid_map) + sizeof(gni_smsg_attr_t));	
	recv_buffer = msg->msg_data;
	dcreg = (struct ptlid_map *)recv_buffer;
	smsg_attr = (gni_smsg_attr_t *)(dcreg+num_cp);
	cur_attr_info = ds->rpc_s->attr_info_start;
	while(1) {
		if(!cur_attr_info->next) {
			cur_attr_info->next = (struct gni_smsg_attr_info *)malloc(sizeof(struct gni_smsg_attr_info));
			memset(cur_attr_info->next, 0, sizeof(struct gni_smsg_attr_info));
			cur_attr_info = cur_attr_info->next;
			cur_attr_info->next = NULL;
			cur_attr_info->appid = dcreg->appid;
			break;
		}
		else {
			cur_attr_info = cur_attr_info->next;	
	    }
    }
	err = rpc_smsg_init(ds->rpc_s, cur_attr_info, num_cp);
	if(err != 0) {
		printf("Rank %d: failed for rpc_smsg_init. (%d)\n", ds->rpc_s->ptlmap.id, err);
		goto err_out;
	}
	peer = &ds->peer_tab[ds->size_sp-1];
	while(1) {
		if(!peer->next) {
			peer->next = (struct node_id *)malloc(num_cp * sizeof(struct node_id));
			memset(peer->next, 0, num_cp * sizeof(struct node_id));
			peer = peer->next;
			break;
		} else {
			peer = (struct node_id *)(peer->next + peer->next->peer_num - 1);
        }        
	}	
	for(j = 0; j < num_cp; j++) {
		peer->ptlmap.nid = dcreg->nid;
		peer->ptlmap.pid = dcreg->pid;
		peer->ptlmap.appid = dcreg->appid;
		peer->ptlmap.id = dcreg->id;		
		peer->peer_rank = j;
		peer->peer_num = num_cp;
		peer->f_reg = 1;
		INIT_LIST_HEAD(&peer->req_list);
		peer->num_msg_at_peer = ds->rpc_s->max_num_msg;
		peer->num_msg_recv = 0;
		peer->num_msg_ret = 0;	  
		peer->sys_msg_recv = 0;
		peer->sys_msg_at_peer = ds->rpc_s->max_num_msg;
		peer->sys_msg_ret = 0;	 
		peer->next = NULL;
		status = GNI_EpCreate(ds->rpc_s->nic_hndl, ds->rpc_s->src_cq_hndl, &peer->ep_hndl);
		if (status != GNI_RC_SUCCESS)
		{
			printf("Fail: GNI_EpCreate returned error. %d.\n", status);
			goto err_free;
		}
		status = GNI_EpBind(peer->ep_hndl, peer->ptlmap.nid, peer->ptlmap.id);
		if (status != GNI_RC_SUCCESS)
		{
			printf("Fail: GNI_EpBind returned error. %d.\n", status);
			goto err_free;
		}
		status = GNI_EpCreate(ds->rpc_s->nic_hndl, ds->rpc_s->sys_cq_hndl, &peer->sys_ep_hndl);
		if (status != GNI_RC_SUCCESS)
		{
			printf("Rank %d: Fail GNI_EpCreate SYS returned error. %d.\n", ds->rpc_s->ptlmap.id, status);
			goto err_out;
		}
		status = GNI_EpBind(peer->sys_ep_hndl, peer->ptlmap.nid, peer->ptlmap.id);
		if (status != GNI_RC_SUCCESS)
		{
		printf("Rank %d: Fail GNI_EpBind SYS returned error. %d.\n", ds->rpc_s->ptlmap.id, status);
		goto err_out;
		}
		peer->remote_smsg_attr = *smsg_attr;	
		err = rpc_smsg_config(ds->rpc_s, cur_attr_info, peer);
		if (err != 0){
			printf("Rank %d: failed for config SMSG for peer %d. (%d)\n", ds->rpc_s->ptlmap.id, peer->ptlmap.id, err);
			goto err_out;
		}	
		dcreg++;
		peer++;
		smsg_attr++;
	}
	dcreg = (struct ptlid_map *)recv_buffer;
	app = app_find(ds, dcreg->appid);
	if (!app) {
		app = app_alloc();
		if (!app) 
			goto err_out;
		ds->num_cp += num_cp;
		app->app_id = dcreg->appid;
		app->app_num_peers = app->app_cnt_peers = num_cp;
		app->app_peer_tab = ds_get_peer(ds, dcreg->id);
		list_add(&app->app_entry, &ds->app_list);
		app->f_reg = 1;
	}
    err = ds_register_cp(ds, app);
	if (err != 0){
		printf("Rank %d: failed for ds_register_cp for application %d. (%d)\n", ds->rpc_s->ptlmap.id, dcreg->appid, err);
		goto err_out;
	}
	free(msg->msg_data);
	free(msg);
    return 0;
err_free:
	printf("'%s()': failed with %d.\n", __func__, status);
	return status;
 err_out:
	printf("(%s): failed. (%d)\n", __func__, err);
	return err;
}
static int sp_register_completion ( struct rpc_server * rpc_s , struct msg_buf * msg ) { struct dart_server * ds = ds_ref_from_rpc ( rpc_s ) ; struct app_info * app ; struct node_id * peer ; int err = - ENOMEM ; int i , j ; struct ptlid_map * dcreg ; struct smsg_attr_reg * sar ; struct gni_smsg_attr_info * cur_attr_info ; gni_smsg_attr_t * smsg_attr ; gni_return_t status ; void * recv_buffer ; int info_size = msg -> size ; int num_cp = info_size / ( sizeof ( struct ptlid_map ) + sizeof ( gni_smsg_attr_t ) ) ; recv_buffer = msg -> msg_data ; dcreg = ( struct ptlid_map * ) recv_buffer ; smsg_attr = ( gni_smsg_attr_t * ) ( dcreg + num_cp ) ; cur_attr_info = ds -> rpc_s -> attr_info_start ; while ( 1 ) { if ( ! cur_attr_info -> next ) { cur_attr_info -> next = ( struct gni_smsg_attr_info * ) malloc ( sizeof ( struct gni_smsg_attr_info ) ) ; memset ( cur_attr_info -> next , 0 , sizeof ( struct gni_smsg_attr_info ) ) ; cur_attr_info = cur_attr_info -> next ; cur_attr_info -> next = NULL ; cur_attr_info -> appid = dcreg -> appid ; break ; } else { cur_attr_info = cur_attr_info -> next ; } } err = rpc_smsg_init ( ds -> rpc_s , cur_attr_info , num_cp ) ; if ( err != 0 ) { printf ( " \n " , ds -> rpc_s -> ptlmap . id , err ) ; goto err_out ; } peer = & ds -> peer_tab [ ds -> size_sp - 1 ] ; while ( 1 ) { if ( ! peer -> next ) { peer -> next = ( struct node_id * ) malloc ( num_cp * sizeof ( struct node_id ) ) ; memset ( peer -> next , 0 , num_cp * sizeof ( struct node_id ) ) ; peer = peer -> next ; break ; } else { peer = ( struct node_id * ) ( peer -> next + peer -> next -> peer_num - 1 ) ; } } for ( j = 0 ; j < num_cp ; j ++ ) { peer -> ptlmap . nid = dcreg -> nid ; peer -> ptlmap . pid = dcreg -> pid ; peer -> ptlmap . appid = dcreg -> appid ; peer -> ptlmap . id = dcreg -> id ; peer -> peer_rank = j ; peer -> peer_num = num_cp ; peer -> f_reg = 1 ; INIT_LIST_HEAD ( & peer -> req_list ) ; peer -> num_msg_at_peer = ds -> rpc_s -> max_num_msg ; peer -> num_msg_recv = 0 ; peer -> num_msg_ret = 0 ; peer -> sys_msg_recv = 0 ; peer -> sys_msg_at_peer = ds -> rpc_s -> max_num_msg ; peer -> sys_msg_ret = 0 ; peer -> next = NULL ; status = GNI_EpCreate ( ds -> rpc_s -> nic_hndl , ds -> rpc_s -> src_cq_hndl , & peer -> ep_hndl ) ; if ( status != GNI_RC_SUCCESS ) { printf ( " \n " , status ) ; goto err_free ; } status = GNI_EpBind ( peer -> ep_hndl , peer -> ptlmap . nid , peer -> ptlmap . id ) ; if ( status != GNI_RC_SUCCESS ) { printf ( " \n " , status ) ; goto err_free ; } status = GNI_EpCreate ( ds -> rpc_s -> nic_hndl , ds -> rpc_s -> sys_cq_hndl , & peer -> sys_ep_hndl ) ; if ( status != GNI_RC_SUCCESS ) { printf ( " \n " , ds -> rpc_s -> ptlmap . id , status ) ; goto err_out ; } status = GNI_EpBind ( peer -> sys_ep_hndl , peer -> ptlmap . nid , peer -> ptlmap . id ) ; if ( status != GNI_RC_SUCCESS ) { printf ( " \n " , ds -> rpc_s -> ptlmap . id , status ) ; goto err_out ; } peer -> remote_smsg_attr = * smsg_attr ; err = rpc_smsg_config ( ds -> rpc_s , cur_attr_info , peer ) ; if ( err != 0 ) { printf ( " \n " , ds -> rpc_s -> ptlmap . id , peer -> ptlmap . id , err ) ; goto err_out ; } dcreg ++ ; peer ++ ; smsg_attr ++ ; } dcreg = ( struct ptlid_map * ) recv_buffer ; app = app_find ( ds , dcreg -> appid ) ; if ( ! app ) { app = app_alloc ( ) ; if ( ! app ) goto err_out ; ds -> num_cp += num_cp ; app -> app_id = dcreg -> appid ; app -> app_num_peers = app -> app_cnt_peers = num_cp ; app -> app_peer_tab = ds_get_peer ( ds , dcreg -> id ) ; list_add ( & app -> app_entry , & ds -> app_list ) ; app -> f_reg = 1 ; } err = ds_register_cp ( ds , app ) ; if ( err != 0 ) { printf ( " \n " , ds -> rpc_s -> ptlmap . id , dcreg -> appid , err ) ; goto err_out ; } free ( msg -> msg_data ) ; free ( msg ) ; return 0 ; err_free : printf ( " \n " , __func__ , status ) ; return status ; err_out : printf ( " \n " , __func__ , err ) ; return err ; }
slave server gets app info and finish local reg for this app
slave server gets app info and finish local reg for this app
//set up new peer_tab block for this app
