title: where does the glibc's implementation of syscall open set ENAMETOOLONG errno when pathname is too long?
body: <p>From the man open: </p>
  
  <pre><code>int open(const char *pathname, int flags);
  
  ...
  
         ENAMETOOLONG
                pathname was too long.
  </code></pre>
  
  <p><code>open</code> will be set <code>errno</code> to <code>ENAMETOOLONG</code>, but I cannot find this assignment in glibc's source code(Maybe it will invoke kernel functions, but I cannot find either)</p>
  
  <p>Below is the functions I have already explored in glibc: </p>
  
  <pre class="lang-c prettyprint-override"><code>/* Open FILE with access OFLAG.  If O_CREAT or O_TMPFILE is in OFLAG,
     a third argument is the file protection.  */
  int
  __libc_open (const char *file, int oflag)
  {
    int mode;
  
    if (file == NULL)
      {
        __set_errno (EINVAL);
        return -1;
      }
  
    if (__OPEN_NEEDS_MODE (oflag))
      {
        va_list arg;
        va_start(arg, oflag);
        mode = va_arg(arg, int);
        va_end(arg);
      }
  
    __set_errno (ENOSYS);
    return -1;
  }
  </code></pre>
  
  <pre class="lang-c prettyprint-override"><code>int
  __open_nocancel (const char *file, int oflag, ...)
  {
    int mode = 0;
  
    if (__OPEN_NEEDS_MODE (oflag))
      {
        va_list arg;
        va_start (arg, oflag);
        mode = va_arg (arg, int);
        va_end (arg);
      }
  
    return INLINE_SYSCALL_CALL (openat, AT_FDCWD, file, oflag, mode);
  }
  </code></pre>
  
  <pre class="lang-c prettyprint-override"><code>/* Open FILE with access OFLAG.  Interpret relative paths relative to
     the directory associated with FD.  If O_CREAT or O_TMPFILE is in OFLAG, a
     third argument is the file protection.  */
  int
  __openat (int fd, const char *file, int oflag, ...)
  {
    int mode;
  
    if (file == NULL)
      {
        __set_errno (EINVAL);
        return -1;
      }
  
    if (fd != AT_FDCWD &amp;&amp; file[0] != '/')
      {
        /* Check FD is associated with a directory.  */
        struct stat64 st;
        if (__fxstat64 (_STAT_VER, fd, &amp;st) != 0)
      return -1;
  
        if (!S_ISDIR (st.st_mode))
      {
        __set_errno (ENOTDIR);
        return -1;
      }
      }
  
    if (__OPEN_NEEDS_MODE (oflag))
      {
        va_list arg;
        va_start (arg, oflag);
        mode = va_arg (arg, int);
        va_end (arg);
  
        ignore_value (mode);
      }
  
    __set_errno (ENOSYS);
    return -1;
  }
  </code></pre>
  
  <pre class="lang-c prettyprint-override"><code>/* Issue a syscall defined by syscall number plus any other argument
     required.  Any error will be handled using arch defined macros and errno
     will be set accordingly.
     It is similar to INLINE_SYSCALL macro, but without the need to pass the
     expected argument number as second parameter.  */
  #define INLINE_SYSCALL_CALL(...) \
    __INLINE_SYSCALL_DISP (__INLINE_SYSCALL, __VA_ARGS__)
  
  #define SYSCALL_CANCEL(...) \
    ({                                         \
      long int sc_ret;                                 \
      if (SINGLE_THREAD_P)                             \
        sc_ret = INLINE_SYSCALL_CALL (__VA_ARGS__);                \
      else                                     \
        {                                      \
      int sc_cancel_oldtype = LIBC_CANCEL_ASYNC ();                \
      sc_ret = INLINE_SYSCALL_CALL (__VA_ARGS__);              \
          LIBC_CANCEL_RESET (sc_cancel_oldtype);                   \
        }                                      \
      sc_ret;                                  \
    })
  </code></pre>