#StackBounty: #c++11 #ubuntu-16.04 #centos7 #zeromq #latency Why does the ZeroMQ ROUTE-DEALER pattern have high latency?

Bounty: 50

Using libzmq 4.2.5 on centos 7. Getting very high latency when messages are sent from DEALER to ROUTER and even from ROUTER to DEALER. So I wrote a simple client-server program using tcp and sent messages between them just for comparison. Tcp appears to be fast.

Sending single byte from DEALER to ROUTER, zmq takes 900 microseconds.

Sending single byte from client to server, tcp takes 150 microseconds.

What am I doing wrong. I thought zmq will be at least as fast as tcp. Is there any tuning I can do to make zmq faster?

Update

router.cpp

#include <zmq.hpp>
    struct data
    {
    char one[21];
    unsigned long two;
   };
data * pdata;
std::size_t counter=0;

 int main()
{
   zmq::context_t context(1);
   zmq::socket_t Device(context,ZMQ_ROUTER);

   int iHighWaterMark=0;

  Device.setsockopt(ZMQ_SNDHWM,&iHighWaterMark,sizeof(int));
  Device.setsockopt(ZMQ_RCVHWM,&iHighWaterMark,sizeof(int));

  Device.bind("tcp://0.0.0.0:5555");

  pdata=new data[10000];

 struct timespec ts_dtime;
 unsigned long sec;

  zmq::message_t message;

  zmq::pollitem_t arrPollItems[]={{Device, 0, ZMQ_POLLIN, 0},{NULL, 
                                                            0, ZMQ_POLLIN, 0}};

    while(counter < 10000)
      {
        try
      {
        int iAssert = zmq::poll(arrPollItems, 1, -1);
        if (iAssert <= 0)
          {
             if (-1 == iAssert)
          {
             printf("zmq_poll failed errno: %d error:%s", errno, 
                                 zmq_strerror(errno));
          }
            continue;
           }

          if (arrPollItems[0].revents == ZMQ_POLLIN)
           {
             while(true)
            {
               if(! Device.recv(&message,ZMQ_DONTWAIT))
                    break;

               Device.recv(&message);

                strncpy(pdata[counter].one, 
                                      (char*)message.data(),message.size());
               clock_gettime(CLOCK_REALTIME, &ts_dtime);
              pdata[counter].two = (ts_dtime.tv_sec*1e9)+ 
                                                              ts_dtime.tv_nsec;
              ++counter;
            }

           }
      }
          catch(...)
        {

         }

          }

         for(int i=0;i<counter;++i)
         printf("%d %s %lun",i+1,pdata[i].one,pdata[i].two);

         return 0;
        }

dealer.cpp

#include <zmq.hpp>
#include<unistd.h>

int main()
{
  zmq::context_t context(1);
  zmq::socket_t Device(context,ZMQ_DEALER);

  int iHighWaterMark=0;

  Device.setsockopt(ZMQ_SNDHWM,&iHighWaterMark,sizeof(int));
  Device.setsockopt(ZMQ_RCVHWM,&iHighWaterMark,sizeof(int));
  Device.setsockopt(ZMQ_IDENTITY,"TEST",4);

   Device.connect("tcp://0.0.0.0:5555");

    usleep(100000);

   struct timespec ts_dtime;
   unsigned long sec;

   for(std::size_t i=0;i<10000;++i)
    {
      clock_gettime(CLOCK_REALTIME, &ts_dtime);
      sec=(ts_dtime.tv_sec*1e9)+ ts_dtime.tv_nsec;
      zmq::message_t message(21);
      sprintf((char *)message.data(),"%lu",sec);
      Device.send(message);
     usleep(500);
    }

  return 0;
 }

update 2:

router.cpp

#include <zmq.hpp>
#include <stdio.h>
#include <stdlib.h>

int main (int argc, char *argv[])
{
    const char *bind_to;
    int roundtrip_count;
    size_t message_size;

    int rc;
    int i;


    if (argc != 4) {
        printf ("usage: local_lat <bind-to> <message-size> "
                "<roundtrip-count>n");
        return 1;
    }
    bind_to = argv[1];
    message_size = atoi (argv[2]);
    roundtrip_count = atoi (argv[3]);

    zmq::context_t ctx(1);
  zmq::socket_t s(ctx,ZMQ_ROUTER);

  zmq::message_t msg,id;

    int iHighWaterMark=0;
    s.setsockopt(ZMQ_SNDHWM , &iHighWaterMark,
                         sizeof (int));
    s.setsockopt(ZMQ_RCVHWM , &iHighWaterMark,
                                              sizeof (int));
    s.bind( bind_to);
    struct timespec ts_dtime;
    unsigned long sec;
for (i = 0; i != roundtrip_count; i++) {
      rc =s.recv(&id);
        if (rc < 0) {
            printf ("error in zmq_recvmsg: %sn", zmq_strerror (errno));
            return -1;
        }

        rc = s.recv(&msg, 0);
        if (rc < 0) {
            printf ("error in zmq_recvmsg: %sn", zmq_strerror (errno));
            return -1;
        }

        clock_gettime(CLOCK_REALTIME, &ts_dtime);
        sec=((ts_dtime.tv_sec*1e9)+ ts_dtime.tv_nsec);
        printf("%.*s %lun",20,(char *)msg.data(),sec);
}




    s.close();



    return 0;
}

dealer.cpp

#include <zmq.hpp>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
int main (int argc, char *argv[])
{
    const char *connect_to;
    int roundtrip_count;
    size_t message_size;

    int rc;
    int i;

    void *watch;
    unsigned long elapsed;
    double latency;

    if (argc != 4) {
        printf ("usage: remote_lat <connect-to> <message-size> "
                "<roundtrip-count>n");
        return 1;
    }
    connect_to = argv[1];
    message_size = atoi (argv[2]);
    roundtrip_count = atoi (argv[3]);

    zmq::context_t ctx(1);
  zmq::socket_t s(ctx,ZMQ_DEALER);

  struct timespec ts_dtime;
  unsigned long sec;
int iHighWaterMark=0;
    s.setsockopt(ZMQ_SNDHWM , &iHighWaterMark,
                         sizeof (int));
    s.setsockopt(ZMQ_RCVHWM , &iHighWaterMark,
                                              sizeof (int));

    s.connect(connect_to);


    for (i = 0; i != roundtrip_count; i++) {
      zmq::message_t msg(message_size+20);
      clock_gettime(CLOCK_REALTIME, &ts_dtime);
      sec=(ts_dtime.tv_sec*1e9)+ ts_dtime.tv_nsec;
      sprintf((char *)msg.data(),"%lu",sec);
      rc = s.send(msg);
        if (rc < 0) {
            printf ("error in zmq_sendmsg: %sn", zmq_strerror (errno));
            return -1;
        }

        sleep(1);
}
s.close();


    return 0;
}

output :

1562125527489432576 1562125527489773568
1562125528489582848 1562125528489961472
1562125529489740032 1562125529490124032
1562125530489944832 1562125530490288896
1562125531490101760 1562125531490439424
1562125532490261248 1562125532490631680
1562125533490422272 1562125533490798080
1562125534490555648 1562125534490980096
1562125535490745856 1562125535491161856
1562125536490894848 1562125536491245824
1562125537491039232 1562125537491416320
1562125538491229184 1562125538491601152
1562125539491375872 1562125539491764736
1562125540491517184 1562125540491908352
1562125541491657984 1562125541492027392
1562125542491816704 1562125542492193536
1562125543491963136 1562125543492338944
1562125544492103680 1562125544492564992
1562125545492248832 1562125545492675328
1562125546492397312 1562125546492783616
1562125547492543744 1562125547492926720
1562125564495211008 1562125564495629824
1562125565495372032 1562125565495783168
1562125566495515904 1562125566495924224
1562125567495660800 1562125567496006144
1562125568495806464 1562125568496160000
1562125569495896064 1562125569496235520
1562125570496080128 1562125570496547584
1562125571496235008 1562125571496666624
1562125572496391424 1562125572496803584
1562125573496532224 1562125573496935680
1562125574496652800 1562125574497053952
1562125575496843776 1562125575497277184
1562125576496997120 1562125576497417216
1562125577497182208 1562125577497726976
1562125578497336832 1562125578497726464
1562125579497549312 1562125579497928704
1562125580497696512 1562125580498115328
1562125581497847808 1562125581498198528
1562125582497998336 1562125582498340096
1562125583498140160 1562125583498622464
1562125584498295296 1562125584498680832
1562125585498445312 1562125585498842624
1562125586498627328 1562125586499025920

All are in the range for 350-450us


Get this bounty!!!

Leave a Reply

This site uses Akismet to reduce spam. Learn how your comment data is processed.