title: Why does the ZeroMQ ROUTER-DEALER pattern have high latency?
body: <p>Using libzmq 4.2.5 on centos 7. Getting very high latency when messages are sent from <code>DEALER</code> to <code>ROUTER</code> and even from <code>ROUTER</code> to <code>DEALER</code>. So I wrote a simple client-server program using tcp and sent messages between them just for comparison. Tcp appears to be fast.</p>
  
  <p>Sending single byte from <code>DEALER</code> to <code>ROUTER</code>, <code>zmq</code> takes 900 microseconds. </p>
  
  <p>Sending single byte from client to server, tcp takes 150 microseconds.</p>
  
  <p>What am I doing wrong. I thought <code>zmq</code> will be at least as fast as tcp. Is there any tuning I can do to make <code>zmq</code> faster?</p>
  
  <p>Update </p>
  
  <p>router.cpp</p>
  
  <pre><code>#include &lt;zmq.hpp&gt;
      struct data
      {
      char one[21];
      unsigned long two;
     };
  data * pdata;
  std::size_t counter=0;
  
   int main()
  {
     zmq::context_t context(1);
     zmq::socket_t Device(context,ZMQ_ROUTER);
  
     int iHighWaterMark=0;
  
    Device.setsockopt(ZMQ_SNDHWM,&amp;iHighWaterMark,sizeof(int));
    Device.setsockopt(ZMQ_RCVHWM,&amp;iHighWaterMark,sizeof(int));
  
    Device.bind("tcp://0.0.0.0:5555");
  
    pdata=new data[10000];
  
   struct timespec ts_dtime;
   unsigned long sec;
  
    zmq::message_t message;
  
    zmq::pollitem_t arrPollItems[]={{Device, 0, ZMQ_POLLIN, 0},{NULL, 
                                                              0, ZMQ_POLLIN, 0}};
  
      while(counter &lt; 10000)
        {
          try
        {
          int iAssert = zmq::poll(arrPollItems, 1, -1);
          if (iAssert &lt;= 0)
            {
               if (-1 == iAssert)
            {
               printf("zmq_poll failed errno: %d error:%s", errno, 
                                   zmq_strerror(errno));
            }
              continue;
             }
  
            if (arrPollItems[0].revents == ZMQ_POLLIN)
             {
               while(true)
              {
                 if(! Device.recv(&amp;message,ZMQ_DONTWAIT))
                      break;
  
                 Device.recv(&amp;message);
  
                  strncpy(pdata[counter].one, 
                                        (char*)message.data(),message.size());
                 clock_gettime(CLOCK_REALTIME, &amp;ts_dtime);
                pdata[counter].two = (ts_dtime.tv_sec*1e9)+ 
                                                                ts_dtime.tv_nsec;
                ++counter;
              }
  
             }
        }
            catch(...)
          {
  
           }
  
            }
  
           for(int i=0;i&lt;counter;++i)
           printf("%d %s %lu\n",i+1,pdata[i].one,pdata[i].two);
  
           return 0;
          }
  </code></pre>
  
  <p>dealer.cpp</p>
  
  <pre><code>#include &lt;zmq.hpp&gt;
  #include&lt;unistd.h&gt;
  
  int main()
  {
    zmq::context_t context(1);
    zmq::socket_t Device(context,ZMQ_DEALER);
  
    int iHighWaterMark=0;
  
    Device.setsockopt(ZMQ_SNDHWM,&amp;iHighWaterMark,sizeof(int));
    Device.setsockopt(ZMQ_RCVHWM,&amp;iHighWaterMark,sizeof(int));
    Device.setsockopt(ZMQ_IDENTITY,"TEST",4);
  
     Device.connect("tcp://0.0.0.0:5555");
  
      usleep(100000);
  
     struct timespec ts_dtime;
     unsigned long sec;
  
     for(std::size_t i=0;i&lt;10000;++i)
      {
        clock_gettime(CLOCK_REALTIME, &amp;ts_dtime);
        sec=(ts_dtime.tv_sec*1e9)+ ts_dtime.tv_nsec;
        zmq::message_t message(21);
        sprintf((char *)message.data(),"%lu",sec);
        Device.send(message);
       usleep(500);
      }
  
    return 0;
   }
  </code></pre>
  
  <p>update 2:</p>
  
  <p>router.cpp</p>
  
  <pre><code>#include &lt;zmq.hpp&gt;
  #include &lt;stdio.h&gt;
  #include &lt;stdlib.h&gt;
  
  int main (int argc, char *argv[])
  {
      const char *bind_to;
      int roundtrip_count;
      size_t message_size;
  
      int rc;
      int i;
  
  
      if (argc != 4) {
          printf ("usage: local_lat &lt;bind-to&gt; &lt;message-size&gt; "
                  "&lt;roundtrip-count&gt;\n");
          return 1;
      }
      bind_to = argv[1];
      message_size = atoi (argv[2]);
      roundtrip_count = atoi (argv[3]);
  
      zmq::context_t ctx(1);
    zmq::socket_t s(ctx,ZMQ_ROUTER);
  
    zmq::message_t msg,id;
  
      int iHighWaterMark=0;
      s.setsockopt(ZMQ_SNDHWM , &amp;iHighWaterMark,
                           sizeof (int));
      s.setsockopt(ZMQ_RCVHWM , &amp;iHighWaterMark,
                                                sizeof (int));
      s.bind( bind_to);
      struct timespec ts_dtime;
      unsigned long sec;
  for (i = 0; i != roundtrip_count; i++) {
        rc =s.recv(&amp;id);
          if (rc &lt; 0) {
              printf ("error in zmq_recvmsg: %s\n", zmq_strerror (errno));
              return -1;
          }
  
          rc = s.recv(&amp;msg, 0);
          if (rc &lt; 0) {
              printf ("error in zmq_recvmsg: %s\n", zmq_strerror (errno));
              return -1;
          }
  
          clock_gettime(CLOCK_REALTIME, &amp;ts_dtime);
          sec=((ts_dtime.tv_sec*1e9)+ ts_dtime.tv_nsec);
          printf("%.*s %lu\n",20,(char *)msg.data(),sec);
  }
  
  
  
  
      s.close();
  
  
  
      return 0;
  }
  </code></pre>
  
  <p>dealer.cpp</p>
  
  <pre><code>#include &lt;zmq.hpp&gt;
  #include &lt;stdio.h&gt;
  #include &lt;stdlib.h&gt;
  #include &lt;string.h&gt;
  #include &lt;unistd.h&gt;
  int main (int argc, char *argv[])
  {
      const char *connect_to;
      int roundtrip_count;
      size_t message_size;
  
      int rc;
      int i;
  
      void *watch;
      unsigned long elapsed;
      double latency;
  
      if (argc != 4) {
          printf ("usage: remote_lat &lt;connect-to&gt; &lt;message-size&gt; "
                  "&lt;roundtrip-count&gt;\n");
          return 1;
      }
      connect_to = argv[1];
      message_size = atoi (argv[2]);
      roundtrip_count = atoi (argv[3]);
  
      zmq::context_t ctx(1);
    zmq::socket_t s(ctx,ZMQ_DEALER);
  
    struct timespec ts_dtime;
    unsigned long sec;
  int iHighWaterMark=0;
      s.setsockopt(ZMQ_SNDHWM , &amp;iHighWaterMark,
                           sizeof (int));
      s.setsockopt(ZMQ_RCVHWM , &amp;iHighWaterMark,
                                                sizeof (int));
  
      s.connect(connect_to);
  
  
      for (i = 0; i != roundtrip_count; i++) {
        zmq::message_t msg(message_size+20);
        clock_gettime(CLOCK_REALTIME, &amp;ts_dtime);
        sec=(ts_dtime.tv_sec*1e9)+ ts_dtime.tv_nsec;
        sprintf((char *)msg.data(),"%lu",sec);
        rc = s.send(msg);
          if (rc &lt; 0) {
              printf ("error in zmq_sendmsg: %s\n", zmq_strerror (errno));
              return -1;
          }
  
          sleep(1);
  }
  s.close();
  
  
      return 0;
  }
  </code></pre>
  
  <p>output :</p>
  
  <pre><code>1562125527489432576 1562125527489773568
  1562125528489582848 1562125528489961472
  1562125529489740032 1562125529490124032
  1562125530489944832 1562125530490288896
  1562125531490101760 1562125531490439424
  1562125532490261248 1562125532490631680
  1562125533490422272 1562125533490798080
  1562125534490555648 1562125534490980096
  1562125535490745856 1562125535491161856
  1562125536490894848 1562125536491245824
  1562125537491039232 1562125537491416320
  1562125538491229184 1562125538491601152
  1562125539491375872 1562125539491764736
  1562125540491517184 1562125540491908352
  1562125541491657984 1562125541492027392
  1562125542491816704 1562125542492193536
  1562125543491963136 1562125543492338944
  1562125544492103680 1562125544492564992
  1562125545492248832 1562125545492675328
  1562125546492397312 1562125546492783616
  1562125547492543744 1562125547492926720
  1562125564495211008 1562125564495629824
  1562125565495372032 1562125565495783168
  1562125566495515904 1562125566495924224
  1562125567495660800 1562125567496006144
  1562125568495806464 1562125568496160000
  1562125569495896064 1562125569496235520
  1562125570496080128 1562125570496547584
  1562125571496235008 1562125571496666624
  1562125572496391424 1562125572496803584
  1562125573496532224 1562125573496935680
  1562125574496652800 1562125574497053952
  1562125575496843776 1562125575497277184
  1562125576496997120 1562125576497417216
  1562125577497182208 1562125577497726976
  1562125578497336832 1562125578497726464
  1562125579497549312 1562125579497928704
  1562125580497696512 1562125580498115328
  1562125581497847808 1562125581498198528
  1562125582497998336 1562125582498340096
  1562125583498140160 1562125583498622464
  1562125584498295296 1562125584498680832
  1562125585498445312 1562125585498842624
  1562125586498627328 1562125586499025920
  </code></pre>
  
  <p>All are in the range for 350-450us</p>