linux ttyname and ttyname_r: do not return wrong results

If a link (say /proc/self/fd/0) pointing to a device, say /dev/pts/2, in a
parent mount namespace is passed to ttyname, and a /dev/pts/2 exists (in a
different devpts) in the current namespace, then it returns /dev/pts/2.
But /dev/pts/2 is NOT the current tty, it is a different file and device.

Detect this case and return ENODEV.  Userspace can choose to take this as a hint
that the fd points to a tty device but to act on the fd rather than the link.

Signed-off-by: Serge Hallyn <serge@hallyn.com>
Signed-off-by: Christian Brauner <christian.brauner@ubuntu.com>

ChangeLog

@@ -1,3 +1,15 @@
+2017-03-19  Christian Brauner  <christian.brauner@ubuntu.com>
+
+	* sysdeps/unix/sysv/linux/ttyname.h: New file.
+	* sysdeps/unix/sysv/linux/ttyname.c: Include "ttyname.h".
+	(ttyname) [!_STATBUF_ST_RDEV]: Make code unconditional.
+	Call is_pty when the link does not exist or does not match, fail
+	with ENODEV when it returns true.
+	* sysdeps/unix/sysv/linux/ttyname_r.c: Include "ttyname.h".
+	(__ttyname_r) [!_STATBUF_ST_RDEV]: Make code unconditional.
+	Call is_pty when the link does not exist or does not match, fail
+	with ENODEV when it returns true.
+
 2017-03-18  Joseph Myers  <joseph@codesourcery.com>
 
 	[BZ #16437]

sysdeps/unix/sysv/linux/ttyname.c

@@ -28,6 +28,8 @@
 
 #include <_itoa.h>
 
+#include "ttyname.h"
+
 #if 0
 /* Is this used anywhere?  It is not exported.  */
 char *__ttyname;
@@ -170,12 +172,18 @@ ttyname (int fd)
 #ifdef _STATBUF_ST_RDEV
 	  && S_ISCHR (st1.st_mode)
 	  && st1.st_rdev == st.st_rdev
-#else
-	  && st1.st_ino == st.st_ino
-	  && st1.st_dev == st.st_dev
 #endif
-	  )
+	  && st1.st_ino == st.st_ino
+	  && st1.st_dev == st.st_dev)
 	return ttyname_buf;
+
+      /* If the link doesn't exist, then it points to a device in another
+	 namespace. */
+      if (is_pty (&st))
+	{
+	  __set_errno (ENODEV);
+	  return NULL;
+	}
     }
 
   if (__xstat64 (_STAT_VER, "/dev/pts", &st1) == 0 && S_ISDIR (st1.st_mode))

sysdeps/unix/sysv/linux/ttyname.h

@@ -0,0 +1,34 @@
+/* Copyright (C) 2016-2017 Free Software Foundation, Inc.
+   This file is part of the GNU C Library.
+
+   The GNU C Library is free software; you can redistribute it and/or
+   modify it under the terms of the GNU Lesser General Public License as
+   published by the Free Software Foundation; either version 2.1 of the
+   License, or (at your option) any later version.
+
+   The GNU C Library is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+   Lesser General Public License for more details.
+
+   You should have received a copy of the GNU Lesser General Public
+   License along with the GNU C Library; see the file COPYING.LIB.  If
+   not, see <http://www.gnu.org/licenses/>.  */
+
+#include <unistd.h>
+#include <sys/sysmacros.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+
+/* Return true if this is a UNIX98 pty device, as defined in
+   linux/Documentation/devices.txt.  */
+static inline int
+is_pty (struct stat64 *sb)
+{
+#ifdef _STATBUF_ST_RDEV
+  int m = major (sb->st_rdev);
+  return (136 <= m && m <= 143);
+#else
+  return false;
+#endif
+}

sysdeps/unix/sysv/linux/ttyname_r.c

@@ -28,6 +28,8 @@
 
 #include <_itoa.h>
 
+#include "ttyname.h"
+
 static int getttyname_r (char *buf, size_t buflen,
 			 dev_t mydev, ino64_t myino, int save,
 			 int *dostat) internal_function;
@@ -152,12 +154,19 @@ __ttyname_r (int fd, char *buf, size_t buflen)
 #ifdef _STATBUF_ST_RDEV
 	  && S_ISCHR (st1.st_mode)
 	  && st1.st_rdev == st.st_rdev
-#else
-	  && st1.st_ino == st.st_ino
-	  && st1.st_dev == st.st_dev
 #endif
-	  )
+	  && st1.st_ino == st.st_ino
+	  && st1.st_dev == st.st_dev)
 	return 0;
+
+      /* If the link doesn't exist, then it points to a device in another
+       * namespace.
+       */
+      if (is_pty (&st))
+	{
+	  __set_errno (ENODEV);
+	  return ENODEV;
+	}
     }
 
   /* Prepare the result buffer.  */