diff --git a/CMakeLists.txt b/CMakeLists.txt
index 63b6baef..c51acd63 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -83,8 +83,6 @@ add_library(format ${FMT_SOURCES})
 if (CMAKE_COMPILER_IS_GNUCXX)
   set_target_properties(format PROPERTIES COMPILE_FLAGS
     "-Wall -Wextra -Wshadow -pedantic")
-elseif (MSVC)
-  set_target_properties(format PROPERTIES COMPILE_FLAGS /W4)
 endif ()
 
 # If FMT_EXTRA_TESTS is TRUE, then test compilation with both -std=c++11
diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt
index 210f3abd..ce2becae 100644
--- a/test/CMakeLists.txt
+++ b/test/CMakeLists.txt
@@ -24,6 +24,10 @@ endif ()
 
 add_fmt_test(gtest-extra-test)
 add_fmt_test(format-test)
+if (FMT_EXTRA_TESTS AND MSVC)
+  set_target_properties(format-test PROPERTIES COMPILE_FLAGS /W4)
+endif ()
+
 add_fmt_test(format-impl-test CUSTOM_LINK)
 add_fmt_test(printf-test)
 foreach (target format-test printf-test)