Linux: Support __IPC_64 in sysvctl *ctl command arguments (bug 29771)

Old applications pass __IPC_64 as part of the command argument because
old glibc did not check for unknown commands, and passed through the
arguments directly to the kernel, without adding __IPC_64.
Applications need to continue doing that for old glibc compatibility,
so this commit enables this approach in current glibc.

For msgctl and shmctl, if no translation is required, make
direct system calls, as we did before the time64 changes.  If
translation is required, mask __IPC_64 from the command argument.

For semctl, the union-in-vararg argument handling means that
translation is needed on all architectures.

Reviewed-by: Adhemerval Zanella  <adhemerval.zanella@linaro.org>
(cherry picked from commit 22a46dee24)

NEWS

@@ -709,6 +709,7 @@ The following bugs are resolved with this release:
   [27237] malloc: deadlock in malloc/tst-malloc-stats-cancellation
   [27256] locale: Assertion failure in ISO-2022-JP-3 gconv module
     related to combining characters (CVE-2021-3326)
+  [29771] Restore IPC_64 support in sysvipc *ctl functions
 
 
 Version 2.32

sysdeps/unix/sysv/linux/ipc_priv.h

@@ -63,4 +63,10 @@ struct __old_ipc_perm
 # define __IPC_TIME64 0
 #endif
 
+#if __IPC_TIME64 || defined __ASSUME_SYSVIPC_BROKEN_MODE_T
+# define IPC_CTL_NEED_TRANSLATION 1
+#else
+# define IPC_CTL_NEED_TRANSLATION 0
+#endif
+
 #include <ipc_ops.h>

sysdeps/unix/sysv/linux/msgctl.c

@@ -86,12 +86,20 @@ msgctl_syscall (int msqid, int cmd, msgctl_arg_t *buf)
 int
 __msgctl64 (int msqid, int cmd, struct __msqid64_ds *buf)
 {
+#if IPC_CTL_NEED_TRANSLATION
 # if __IPC_TIME64
   struct kernel_msqid64_ds ksemid, *arg = NULL;
 # else
   msgctl_arg_t *arg;
 # endif
 
+  /* Some applications pass the __IPC_64 flag in cmd, to invoke
+     previously unsupported commands back when there was no EINVAL
+     error checking in glibc.  Mask the flag for the switch statements
+     below.  msgctl_syscall adds back the __IPC_64 flag for the actual
+     system call.  */
+  cmd &= ~__IPC_64;
+
   switch (cmd)
     {
     case IPC_RMID:
@@ -153,6 +161,10 @@ __msgctl64 (int msqid, int cmd, struct __msqid64_ds *buf)
     }
 
   return ret;
+
+#else /* !IPC_CTL_NEED_TRANSLATION */
+  return msgctl_syscall (msqid, cmd, buf);
+#endif
 }
 #if __TIMESIZE != 64
 libc_hidden_def (__msgctl64)

sysdeps/unix/sysv/linux/semctl.c

@@ -141,6 +141,13 @@ __semctl64 (int semid, int semnum, int cmd, ...)
   union semun64 arg64 = { 0 };
   va_list ap;
 
+  /* Some applications pass the __IPC_64 flag in cmd, to invoke
+     previously unsupported commands back when there was no EINVAL
+     error checking in glibc.  Mask the flag for the switch statements
+     below.  semctl_syscall adds back the __IPC_64 flag for the actual
+     system call.  */
+  cmd &= ~__IPC_64;
+
   /* Get the argument only if required.  */
   switch (cmd)
     {

sysdeps/unix/sysv/linux/shmctl.c

@@ -86,12 +86,20 @@ shmctl_syscall (int shmid, int cmd, shmctl_arg_t *buf)
 int
 __shmctl64 (int shmid, int cmd, struct __shmid64_ds *buf)
 {
+#if IPC_CTL_NEED_TRANSLATION
 # if __IPC_TIME64
   struct kernel_shmid64_ds kshmid, *arg = NULL;
 # else
   shmctl_arg_t *arg;
 # endif
 
+  /* Some applications pass the __IPC_64 flag in cmd, to invoke
+     previously unsupported commands back when there was no EINVAL
+     error checking in glibc.  Mask the flag for the switch statements
+     below.  shmctl_syscall adds back the __IPC_64 flag for the actual
+     system call.  */
+  cmd &= ~__IPC_64;
+
   switch (cmd)
     {
     case IPC_RMID:
@@ -156,6 +164,10 @@ __shmctl64 (int shmid, int cmd, struct __shmid64_ds *buf)
     }
 
   return ret;
+
+#else /* !IPC_CTL_NEED_TRANSLATION */
+  return shmctl_syscall (shmid, cmd, buf);
+#endif
 }
 #if __TIMESIZE != 64
 libc_hidden_def (__shmctl64)