Bug 20918 - Building with --enable-nss-crypt fails tst-linkall-static

Some configurations may use NSS cryptographic routines but have no
static library for those routines. The following changes allow glibc to
be built and tested with --enable-nss-crypt, but without having a static
NSS library. At a high level the change does two things:

(1) Detect at configure time if static NSS crypto libraries are
    available.  Assumes libfreebl3.a (instead of the existing Fedora
    libfreebl.a which is incomplete) which matches libfreebl3.so.

(2) If static NSS crypto libraries are _not_ available then adjust the
    way in which we build tst-linkall-static. This includes excluding a
    reference to crypt and not linking against libcrypt.a, all of which
    will fail otherwise.

Testing assumptions:
* Static library is named libfreebl3.a (not libfreebl.a as is currently
  provided in Fedora), matching libfreebl3.so shared link name.

Tested on x86_64 on Fedora with:

(a) --enable-nss-crypt, with no static NSS library support: PASS
    (previous FAIL)

(b) --enable-nss-crypt, with faked static NSS library support: PASS
    (unsupported)
	* Requires changing elf/Makefile to include a stub
	  /lib64/libfreebl3.a for testing purposes.

(c) --disable-nss-crypt: PASS
    (default)

No regressions on x86_64.

For details see:
https://www.sourceware.org/ml/libc-alpha/2016-11/msg00647.html

ChangeLog

@@ -1,3 +1,19 @@
+2016-12-02  Carlos O'Donell  <carlos@redhat.com>
+
+	[BZ #20918]
+	* configure.ac: Test for static NSS cryptographic libraries and set
+	libc_cv_static_nss_crypt.
+	* configure: Regenerate.
+	* config.make.in (static-nss-crypt): Define.
+	* elf/Makefile (CFLAGS-tst-linkall-static.c): Define.
+	[ifeq (yesno,$(nss-crypt)$(static-nss-crypt))]
+	(CFLAGS-tst-linkall-static.c): Define.
+	($(objpfx)tst-linkall-static): Remove libcrypt.a.
+	[ifeq (yesyes,$(nss-crypt)$(static-nss-crypt))]
+	($(objpfx)tst-linkall-static): Define.
+	[ifeq (no,$(nss-crypt))] ($(objpfx)tst-linkall-static): Define.
+	* elf/tst-linkall-static.c [USE_CRYPT](references): Reference crypt().
+
 2016-12-02  Florian Weimer  <fweimer@redhat.com>
 
 	* elf/Makefile [build-shared] (tests): Add tst-latepthread.

config.make.in

@@ -75,6 +75,7 @@ multi-arch = @multi_arch@
 mach-interface-list = @mach_interface_list@
 
 nss-crypt = @libc_cv_nss_crypt@
+static-nss-crypt = @libc_cv_static_nss_crypt@
 
 # Configuration options.
 build-shared = @shared@

configure

@@ -665,6 +665,7 @@ add_ons
 build_pt_chown
 build_nscd
 link_obsolete_rpc
+libc_cv_static_nss_crypt
 libc_cv_nss_crypt
 enable_werror
 all_warnings
@@ -3529,6 +3530,7 @@ cannot find NSS headers with lowlevel hash function interfaces" "$LINENO" 5
 fi
 rm -f core conftest.err conftest.$ac_objext conftest.$ac_ext
   old_LIBS="$LIBS"
+  old_LDFLAGS="$LDFLAGS"
   LIBS="$LIBS -lfreebl3"
   cat confdefs.h - <<_ACEOF >conftest.$ac_ext
 /* end confdefs.h.  */
@@ -3551,14 +3553,41 @@ cannot link program using lowlevel NSS hash functions" "$LINENO" 5
 fi
 rm -f core conftest.err conftest.$ac_objext \
     conftest$ac_exeext conftest.$ac_ext
+  # Check to see if there is a static NSS cryptographic library.
+  # If there isn't then we can't link anything with libcrypt.a,
+  # and that might mean disabling some static tests.
+  LDFLAGS="$LDFLAGS -static"
+  cat confdefs.h - <<_ACEOF >conftest.$ac_ext
+/* end confdefs.h.  */
+typedef int PRBool;
+#include <hasht.h>
+#include <nsslowhash.h>
+int
+main ()
+{
+NSSLOW_Init();
+  ;
+  return 0;
+}
+_ACEOF
+if ac_fn_c_try_link "$LINENO"; then :
+  libc_cv_static_nss_crypt=yes
+else
+  libc_cv_static_nss_crypt=no
+fi
+rm -f core conftest.err conftest.$ac_objext \
+    conftest$ac_exeext conftest.$ac_ext
+  LDFLAGS="$old_LDFLAGS"
   CFLAGS="$old_CFLAGS"
   LIBS="$old_LIBS"
 else
   libc_cv_nss_crypt=no
+  libc_cv_static_nss_crypt=no
 fi
 
 
 
+
 # Check whether --enable-obsolete-rpc was given.
 if test "${enable_obsolete_rpc+set}" = set; then :
   enableval=$enable_obsolete_rpc; link_obsolete_rpc=$enableval

configure.ac

@@ -321,6 +321,7 @@ void f (void) { NSSLOW_Init (); }])],
 	     AC_MSG_ERROR([
 cannot find NSS headers with lowlevel hash function interfaces]))
   old_LIBS="$LIBS"
+  old_LDFLAGS="$LDFLAGS"
   LIBS="$LIBS -lfreebl3"
   AC_LINK_IFELSE([AC_LANG_PROGRAM([typedef int PRBool;
 #include <hasht.h>
@@ -329,12 +330,25 @@ cannot find NSS headers with lowlevel hash function interfaces]))
 		 libc_cv_nss_crypt=yes,
 		 AC_MSG_ERROR([
 cannot link program using lowlevel NSS hash functions]))
+  # Check to see if there is a static NSS cryptographic library.
+  # If there isn't then we can't link anything with libcrypt.a,
+  # and that might mean disabling some static tests.
+  LDFLAGS="$LDFLAGS -static"
+  AC_LINK_IFELSE([AC_LANG_PROGRAM([typedef int PRBool;
+#include <hasht.h>
+#include <nsslowhash.h>],
+				  [NSSLOW_Init();])],
+		 libc_cv_static_nss_crypt=yes,
+		 libc_cv_static_nss_crypt=no)
+  LDFLAGS="$old_LDFLAGS"
   CFLAGS="$old_CFLAGS"
   LIBS="$old_LIBS"
 else
   libc_cv_nss_crypt=no
+  libc_cv_static_nss_crypt=no
 fi
 AC_SUBST(libc_cv_nss_crypt)
+AC_SUBST(libc_cv_static_nss_crypt)
 
 
 AC_ARG_ENABLE([obsolete-rpc],

elf/Makefile

@@ -337,6 +337,16 @@ $(objpfx)tst-_dl_addr_inside_object: $(objpfx)dl-addr-obj.os
 CFLAGS-tst-_dl_addr_inside_object.c += $(PIE-ccflag)
 endif
 
+# By default tst-linkall-static should try to use crypt routines to test
+# static libcrypt use.
+CFLAGS-tst-linkall-static.c = -DUSE_CRYPT=1
+# However, if we are using NSS crypto and we don't have a static
+# library, then we exclude the use of crypt functions in the test.
+# We similarly exclude libcrypt.a from the static link (see below).
+ifeq (yesno,$(nss-crypt)$(static-nss-crypt))
+CFLAGS-tst-linkall-static.c = -DUSE_CRYPT=0
+endif
+
 include ../Rules
 
 ifeq (yes,$(build-shared))
@@ -1307,12 +1317,30 @@ $(objpfx)tst-ldconfig-X.out : tst-ldconfig-X.sh $(objpfx)ldconfig
 
 $(objpfx)tst-dlsym-error: $(libdl)
 
+# Test static linking of all the libraries we can possibly link
+# together.  Note that in some configurations this may be less than the
+# complete list of libraries we build but we try to maxmimize this list.
 $(objpfx)tst-linkall-static: \
   $(common-objpfx)math/libm.a \
-  $(common-objpfx)crypt/libcrypt.a \
   $(common-objpfx)resolv/libresolv.a \
   $(common-objpfx)dlfcn/libdl.a \
   $(common-objpfx)login/libutil.a \
   $(common-objpfx)rt/librt.a \
   $(common-objpfx)resolv/libanl.a \
-  $(static-thread-library) \
+  $(static-thread-library)
+
+# If we are using NSS crypto and we have the ability to link statically
+# then we include libcrypt.a, otherwise we leave out libcrypt.a and
+# link as much as we can into the tst-linkall-static test.  This assumes
+# that linking with libcrypt.a does everything required to include the
+# static NSS crypto library.
+ifeq (yesyes,$(nss-crypt)$(static-nss-crypt))
+$(objpfx)tst-linkall-static: \
+  $(common-objpfx)crypt/libcrypt.a
+endif
+# If we are not using NSS crypto then we always have the ability to link
+# with libcrypt.a.
+ifeq (no,$(nss-crypt))
+$(objpfx)tst-linkall-static: \
+  $(common-objpfx)crypt/libcrypt.a
+endif

elf/tst-linkall-static.c

@@ -32,7 +32,9 @@ void *references[] =
   {
     &pow,                       /* libm */
     &pthread_create,            /* libpthread */
+#if USE_CRYPT
     &crypt,                     /* libcrypt */
+#endif
     &res_send,                  /* libresolv */
     &dlopen,                    /* libdl */
     &login,                     /* libutil */