title: How to use socat proxy to intercept Unix domain socket sending ancillary data?
body: <p>I have two programs, a server and a client. The server opens a file, writes data to it, and then send its file descriptor to the client over a unix domain socket. Everything works fine untill I introduce a socat proxy in between.</p>
  
  <pre><code>socat -x -v UNIX-LISTEN:/tmp/unixSockSendFe,mode=775,reuseaddr,fork UNIX-CONNECT:/tmp/unixSockSendFd 
  </code></pre>
  
  <p><strong>Explanation</strong>
  <br />  The server listens on <code>/tmp/unixSockSendFd</code>, socat connects to it(<code>UNIX-CONNECT:/tmp/unixSockSendFd</code>), and creates another Unix domain socket(<code>UNIX-LISTEN:/tmp/unixSockSendFe,mode=775,reuseaddr,fork</code>), on which the client connects. Any communication between the client and server gets relayed through <code>socat</code>, which prints the bytes sent in their binary (<code>-x</code> option), and ascii (<code>-v</code> option) form.</p>
  
  <p>If I don't use socat, and client directly connects to server(on  <code>/tmp/unixSockSendFd</code> socket), everything works fine, but when socat is used as a proxy, the client crashes with a segmentation fault.</p>
  
  <p><strong>Server</strong></p>
  
  <pre><code>/*Server code - sendfd.c*/
  #include &lt;sys/socket.h&gt;
  #include &lt;sys/un.h&gt;
  #include &lt;stdio.h&gt;
  #include &lt;stdlib.h&gt;
  #include &lt;unistd.h&gt;
  #include &lt;sys/types.h&gt;
  #include &lt;fcntl.h&gt;
  #include &lt;sys/stat.h&gt;
  #include &lt;errno.h&gt;
  
  char *socket_path = "/tmp/unixSockSendFd";
  char *file="/tmp/abcd.txt" ;/*file whose fd is to be sent*/
  int sendfd(int sock, int fd);
  int recvfd(int s);
  char data[]="sahil\0";
  
  int main(int argc, char *argv[]) {
    struct sockaddr_un addr;
    char buf[100];
          buf[0]='\n';
    int fd,rc,confd;
  
  int fd_to_send;
  int temp,len;
  temp=1;
  
  fd_to_send=open(file,O_TRUNC|O_RDWR|O_CREAT,S_IRWXU|S_IRWXG|S_IRWXO);
  if(fd_to_send==-1)
  {
  perror("file open error");
  return -1;
  }
  
    if (argc &gt; 1) socket_path=argv[1];
  
    if ( (fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
      perror("socket error");
      exit(-1);
    }
  
    memset(&amp;addr, 0, sizeof(addr));
    addr.sun_family = AF_UNIX;
    if (*socket_path == '\0') {
      *addr.sun_path = '\0';
      strncpy(addr.sun_path+1, socket_path+1, sizeof(addr.sun_path)-2);
    } else {
      strncpy(addr.sun_path, socket_path, sizeof(addr.sun_path)-1);
    }
  unlink(socket_path);
  
  if(bind(fd,(struct sockaddr*)&amp;addr,sizeof(addr))==-1){
  perror("bind error");
  return -1;
  }
  
  /*Writing data to file before sending fd*/
  len=write(fd_to_send,data,(int)strlen(data));
  fsync(fd_to_send);
  printf("(len=%d)data written in file(content between ## marks) ##%s##\n",len,data);
  
  listen(fd,1);
  for(;;){
  
  confd=accept(fd,NULL,NULL);
  if(confd==-1)
  {
  perror("accept error");
  continue;
  }
  else{
  printf("new client connected ... sending fd ... \n");
  sendfd(confd,fd_to_send);
  close(confd);
  }
  }
  
    return 0;
  }
  
  int sendfd(int sock, int fd)
  {
    struct msghdr hdr;
    struct iovec data;
  
    char cmsgbuf[CMSG_SPACE(sizeof(int))];
  
    char dummy = '*';
    data.iov_base = &amp;dummy;
    data.iov_len = sizeof(dummy);
  
    memset(&amp;hdr, 0, sizeof(hdr));
    hdr.msg_name = NULL;
    hdr.msg_namelen = 0;
    hdr.msg_iov = &amp;data;
    hdr.msg_iovlen = 1;
    hdr.msg_flags = 0;
  
    hdr.msg_control = cmsgbuf;
    hdr.msg_controllen = CMSG_LEN(sizeof(int));
  
    struct cmsghdr* cmsg = CMSG_FIRSTHDR(&amp;hdr);
    cmsg-&gt;cmsg_len   = CMSG_LEN(sizeof(int));
    cmsg-&gt;cmsg_level = SOL_SOCKET;
    cmsg-&gt;cmsg_type  = SCM_RIGHTS;
  
    *(int*)CMSG_DATA(cmsg) = fd;
  //memcpy((CMSG_DATA(cmsg)), &amp;fd, sizeof(fd)); -- from ivshmem server code - this too works instead of previous line 
  
    int n = sendmsg(sock, &amp;hdr, 0);
  
    if(n == -1)
      printf("sendmsg() failed: %s (socket fd = %d)\n", strerror(errno), sock);
  
    return n;
  }
  
  
  int recvfd(int s)
  {
          int n;
          int fd;
          char buf[1];
          struct iovec iov;
          struct msghdr msg;
          struct cmsghdr *cmsg;
          char cms[CMSG_SPACE(sizeof(int))];
  
          iov.iov_base = buf;
          iov.iov_len = 1;
  
          memset(&amp;msg, 0, sizeof msg);
          msg.msg_name = 0;
          msg.msg_namelen = 0;
          msg.msg_iov = &amp;iov;
          msg.msg_iovlen = 1;
  
          msg.msg_control = (caddr_t)cms;
          msg.msg_controllen = sizeof cms;
  
          if((n=recvmsg(s, &amp;msg, 0)) &lt; 0)
                  return -1;
          if(n == 0){
                  perror("unexpected EOF");
                  return -1;
          }
          cmsg = CMSG_FIRSTHDR(&amp;msg);
          memmove(&amp;fd, CMSG_DATA(cmsg), sizeof(int));
          return fd;
  }
  </code></pre>
  
  <p><strong>Client</strong></p>
  
  <pre><code>/*Client code - recvfd.c*/
  #include &lt;sys/socket.h&gt;
  #include &lt;sys/un.h&gt;
  #include &lt;stdio.h&gt;
  #include &lt;stdlib.h&gt;
  #include &lt;unistd.h&gt;
  #include &lt;sys/types.h&gt;
  #include &lt;fcntl.h&gt;
  #include &lt;sys/stat.h&gt;
  #include &lt;errno.h&gt;
  
  char *socket_path = "/tmp/unixSockSendFe";
  int sendfd(int sock, int fd);
  int recvfd(int s);
  int fd_received;
  
  int main(int argc, char *argv[]) {
    struct sockaddr_un addr;
    char buf[100];
          buf[0]='\n';
    int fd,rc,confd;
  
  int temp,len;
  temp=1;
  
    if (argc &gt; 1) socket_path=argv[1];
  
    if ( (fd = socket(AF_UNIX, SOCK_STREAM, 0)) == -1) {
      perror("socket error");
      exit(-1);
    }
  
    memset(&amp;addr, 0, sizeof(addr));
    addr.sun_family = AF_UNIX;
    if (*socket_path == '\0') {
      *addr.sun_path = '\0';
      strncpy(addr.sun_path+1, socket_path+1, sizeof(addr.sun_path)-2);
    } else {
      strncpy(addr.sun_path, socket_path, sizeof(addr.sun_path)-1);
    }
  
  if(connect(fd,(struct sockaddr*)&amp;addr,sizeof(addr))==-1)
  {
  perror("connect error");
  exit(-1);
  }
  
  fd_received=recvfd(fd);
  lseek(fd_received,0,SEEK_SET);
  len=read(fd_received,buf,5);
  if(len&lt;0)
  {
  perror("read error");
  }
  printf("(fd_received=%d,len=%d) first %d characters read from the file whoes fd was received(content within ##) ##%.*s##\n",fd_received,len,5,5,buf);
  
  
  
    return 0;
  }
  
  int sendfd(int sock, int fd)
  {
    struct msghdr hdr;
    struct iovec data;
  
    char cmsgbuf[CMSG_SPACE(sizeof(int))];
  
    char dummy = '*';
    data.iov_base = &amp;dummy;
    data.iov_len = sizeof(dummy);
  
    memset(&amp;hdr, 0, sizeof(hdr));
    hdr.msg_name = NULL;
    hdr.msg_namelen = 0;
    hdr.msg_iov = &amp;data;
    hdr.msg_iovlen = 1;
    hdr.msg_flags = 0;
  
    hdr.msg_control = cmsgbuf;
    hdr.msg_controllen = CMSG_LEN(sizeof(int));
  
    struct cmsghdr* cmsg = CMSG_FIRSTHDR(&amp;hdr);
    cmsg-&gt;cmsg_len   = CMSG_LEN(sizeof(int));
    cmsg-&gt;cmsg_level = SOL_SOCKET;
    cmsg-&gt;cmsg_type  = SCM_RIGHTS;
  
    *(int*)CMSG_DATA(cmsg) = fd;
  
    int n = sendmsg(sock, &amp;hdr, 0);
  
    if(n == -1)
      printf("sendmsg() failed: %s (socket fd = %d)\n", strerror(errno), sock);
  
    return n;
  }
  
  
  int recvfd(int s)
  {
          int n;
          int fd;
          char buf[1];
          struct iovec iov;
          struct msghdr msg;
          struct cmsghdr *cmsg;
          char cms[CMSG_SPACE(sizeof(int))];
  
          iov.iov_base = buf;
          iov.iov_len = 1;
  
          memset(&amp;msg, 0, sizeof msg);
          msg.msg_name = 0;
          msg.msg_namelen = 0;
          msg.msg_iov = &amp;iov;
          msg.msg_iovlen = 1;
  
          msg.msg_control = (caddr_t)cms;
          msg.msg_controllen = sizeof cms;
  
          if((n=recvmsg(s, &amp;msg, 0)) &lt; 0)
                  return -1;
          if(n == 0){
                  perror("unexpected EOF");
                  return -1;
          }
          cmsg = CMSG_FIRSTHDR(&amp;msg);
          memmove(&amp;fd, CMSG_DATA(cmsg), sizeof(int));
          return fd;
  }
  </code></pre>
  
  <p>On running client (recvfd) I get segmentation fault.</p>
  
  <pre><code>./recvfd          
  [1]    6104 segmentation fault (core dumped)  ./recvfd
  </code></pre>
  
  <p>Here are lines from running gdb with coredump</p>
  
  <pre><code>Core was generated by `./recvfd'.
  Program terminated with signal SIGSEGV, Segmentation fault.
  #0  0x0000000000400cf9 in recvfd (s=3) at recvfd.c:146
  146         memmove(&amp;fd, CMSG_DATA(cmsg), sizeof(int));
  </code></pre>
  
  <p>Here is the core dump - <a href="https://github.com/scratchpadRepos/unixSocketSendFD-log/blob/master/core" rel="noreferrer">Link</a>.</p>
  
  <p>I want to sniff the communication happening between the two processes when the file descriptor is being sent. I am not able to figure out why the client crashes when run with socat, but doesn't when run without it.</p>
  
  <p><strong>Update 1</strong></p>
  
  <p>While using <code>socat</code> to sniff communication happening between two processes of a well established open source project (<strong>ivshmem</strong> - used for sharing memory between running virtual machines, also a part of Intel DPDK, <a href="http://dpdk.org/doc/guides-16.04/prog_guide/ivshmem_lib.html" rel="noreferrer">Link</a>), I observed the following.</p>
  
  <ul>
  <li>None of the processes crash on using <code>socat</code></li>
  <li>When <code>socat</code> is used, the file descriptor is not properly sent, and does not get added to the recipient process.</li>
  <li>When <code>socat</code> is not used, and the two processes are connected directly, the file descriptor gets sent properly, and gets added to the recipient process.</li>
  </ul>