Automated testing tweaks for ObjC

- Move the ObjC tests into the list and exclude them on linux, this will change where in the order they start, since they are longer, it will have other things run in parallel instead of them ending up last and taking the longest. - Switch to the Xcode 7.3 image. - Drop the use of xctool and stream line things through the full_mac_build.sh script. This means we end up with only one build script instead of two. - Tweaks to the mac build script: - Make iOS Xcode version support explicit - Support Debug/Release only building - Change the OS X min parallel count to 2 to better deal with VMs. - Split the travis ios tests into the two Xcode Configurations as the logs are choking travis.
2016-05-24 11:27:33 -04:00 · 2016-05-24 11:27:33 -04:00 · 368a2f4cee
commit 368a2f4cee
parent 2131b2d544
3 changed files with 91 additions and 86 deletions
--- a/.travis.yml
+++ b/.travis.yml
@ -8,7 +8,7 @@ os:
  - linux
  - osx
 # The Objective C build needs Xcode 7.0 or later.
-osx_image: xcode7.2
+osx_image: xcode7.3
 script:
  - ./tests.sh $CONFIG
 env:
@ -23,6 +23,11 @@ env:
  - CONFIG=javanano_jdk7
  - CONFIG=javanano_oracle7
  - CONFIG=javascript
+  # iOS build log was starting to choke travis UI, so split to cover the
+  # Xcode Debug and Release Configurations independently.
+  - CONFIG=objectivec_ios_debug
+  - CONFIG=objectivec_ios_release
+  - CONFIG=objectivec_osx
  - CONFIG=python
  - CONFIG=python_cpp
  - CONFIG=ruby19
@ -54,12 +59,13 @@ matrix:
    # which doesn't work on OS X.
    - os: osx
      env: CONFIG=golang
-  # Add into the matrix OS X tests of Objective C (needs Xcode, so it won't
-  # work on other platforms). These are split so it doesn't take as long to run.
-  include:
-    - os: osx
-      env: CONFIG=objectivec_ios
-    - os: osx
+    # OS X/iOS tests of Objective C (needs Xcode, so it won't work on other
+    # platforms).
+    - os: linux
+      env: CONFIG=objectivec_ios_debug
+    - os: linux
+      env: CONFIG=objectivec_ios_release
+    - os: linux
      env: CONFIG=objectivec_osx
  allow_failures:
    # These currently do not work on OS X but are being worked on by @haberman.
@ -71,14 +77,11 @@ matrix:
    # we moved to an OS X image that is 10.11.
    - os: osx
      env: CONFIG=python_cpp
-    # xctool 0.2.8 seems to have a bug where it randomly kills tests saying
-    # they failed.
-    #   https://github.com/facebook/xctool/issues/619
-    #   https://github.com/google/protobuf/issues/1232
-    # travis updated their images to include 0.2.8:
-    #   https://blog.travis-ci.com/2016-03-23-xcode-image-updates
-    # Mark the iOS test as flakey so these failures don't turn things red.
+    # Mark the iOS test as flakey as xcodebuild some times fails to start the
+    # iOS Simulator.
    - os: osx
-      env: CONFIG=objectivec_ios
+      env: CONFIG=objectivec_ios_debug
+    - os: osx
+      env: CONFIG=objectivec_ios_release
 notifications:
  email: false
--- a/objectivec/DevTools/full_mac_build.sh
+++ b/objectivec/DevTools/full_mac_build.sh
@ -37,6 +37,10 @@ OPTIONS:
         Skip the invoke of Xcode to test the runtime on both iOS and OS X.
   --skip-xcode-ios
         Skip the invoke of Xcode to test the runtime on iOS.
+   --skip-xcode-debug
+         Skip the Xcode Debug configuration.
+   --skip-xcode-release
+         Skip the Xcode Release configuration.
   --skip-xcode-osx
         Skip the invoke of Xcode to test the runtime on OS X.
   --skip-objc-conformance
@ -66,8 +70,8 @@ wrapped_make() {
 }

 NUM_MAKE_JOBS=$(/usr/sbin/sysctl -n hw.ncpu)
-if [[ "${NUM_MAKE_JOBS}" -lt 4 ]] ; then
-  NUM_MAKE_JOBS=4
+if [[ "${NUM_MAKE_JOBS}" -lt 2 ]] ; then
+  NUM_MAKE_JOBS=2
 fi

 DO_AUTOGEN=no
@ -76,6 +80,8 @@ REGEN_DESCRIPTORS=no
 CORE_ONLY=no
 DO_XCODE_IOS_TESTS=yes
 DO_XCODE_OSX_TESTS=yes
+DO_XCODE_DEBUG=yes
+DO_XCODE_RELEASE=yes
 DO_OBJC_CONFORMANCE_TESTS=yes
 while [[ $# != 0 ]]; do
  case "${1}" in
@ -109,6 +115,12 @@ while [[ $# != 0 ]]; do
    --skip-xcode-osx )
      DO_XCODE_OSX_TESTS=no
      ;;
+    --skip-xcode-debug )
+      DO_XCODE_DEBUG=no
+      ;;
+    --skip-xcode-release )
+      DO_XCODE_RELEASE=no
+      ;;
    --skip-objc-conformance )
      DO_OBJC_CONFORMANCE_TESTS=no
      ;;
@ -151,18 +163,26 @@ if [[ "${DO_CLEAN}" == "yes" ]] ; then
        -project objectivec/ProtocolBuffers_iOS.xcodeproj
        -scheme ProtocolBuffers
    )
+    if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then
      "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Debug clean
+    fi
+    if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then
      "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Release clean
    fi
+  fi
  if [[ "${DO_XCODE_OSX_TESTS}" == "yes" ]] ; then
    XCODEBUILD_CLEAN_BASE_OSX=(
      xcodebuild
        -project objectivec/ProtocolBuffers_OSX.xcodeproj
        -scheme ProtocolBuffers
    )
+    if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then
      "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Debug clean
+    fi
+    if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then
      "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Release clean
    fi
+  fi
 fi

 if [[ "${REGEN_DESCRIPTORS}" == "yes" ]] ; then
@ -222,6 +242,14 @@ if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then
          -destination "platform=iOS Simulator,name=iPad Air,OS=9.0" # 64bit
      )
      ;;
+    7.2* )
+      XCODEBUILD_TEST_BASE_IOS+=(
+          -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit
+          -destination "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit
+          -destination "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit
+          -destination "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit
+      )
+      ;;
    7.3* )
      XCODEBUILD_TEST_BASE_IOS+=(
          -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit
@ -230,23 +258,19 @@ if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then
          -destination "platform=iOS Simulator,name=iPad Air,OS=9.3" # 64bit
      )
      ;;
-    7.* )
-      XCODEBUILD_TEST_BASE_IOS+=(
-          -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit
-          -destination "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit
-          -destination "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit
-          -destination "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit
-      )
-      ;;
    * )
      echo "Time to update the simulator targets for Xcode ${XCODE_VERSION}"
      exit 2
      ;;
  esac
+  if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then
    header "Doing Xcode iOS build/tests - Debug"
    "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Debug test
+  fi
+  if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then
    header "Doing Xcode iOS build/tests - Release"
    "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Release test
+  fi
  # Don't leave the simulator in the developer's face.
  killall "${IOS_SIMULATOR_NAME}"
 fi
@ -258,13 +282,18 @@ if [[ "${DO_XCODE_OSX_TESTS}" == "yes" ]] ; then
      # Since the ObjC 2.0 Runtime is required, 32bit OS X isn't supported.
      -destination "platform=OS X,arch=x86_64" # 64bit
  )
+  if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then
    header "Doing Xcode OS X build/tests - Debug"
    "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Debug test
+  fi
+  if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then
    header "Doing Xcode OS X build/tests - Release"
    "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Release test
+  fi
 fi

 if [[ "${DO_OBJC_CONFORMANCE_TESTS}" == "yes" ]] ; then
+  header "Running ObjC Conformance Tests"
  cd conformance
  wrapped_make -j "${NUM_MAKE_JOBS}" test_objc
  cd ..
--- a/tests.sh
+++ b/tests.sh
@ -197,59 +197,30 @@ internal_install_python_deps() {
  fi
 }

-internal_objectivec_common () {
-  # Make sure xctool is up to date. Adapted from
-  #  http://docs.travis-ci.com/user/osx-ci-environment/
-  # We don't use a before_install because we test multiple OSes.
-  brew update
-  brew outdated xctool || brew upgrade xctool
-  # Reused the build script that takes care of configuring and ensuring things
-  # are up to date. Xcode and conformance tests will be directly invoked.
-  objectivec/DevTools/full_mac_build.sh \
-      --core-only --skip-xcode --skip-objc-conformance
-}
-
-internal_xctool_debug_and_release() {
-  # Always use -reporter plain to avoid escape codes in output (makes travis
-  # logs easier to read).
-  xctool -reporter plain -configuration Debug "$@"
-  xctool -reporter plain -configuration Release "$@"
-}
-
 build_objectivec_ios() {
-  internal_objectivec_common
-  # https://github.com/facebook/xctool/issues/509 - unlike xcodebuild, xctool
-  # doesn't support >1 destination, so we have to build first and then run the
-  # tests one destination at a time.
-  internal_xctool_debug_and_release \
-    -project objectivec/ProtocolBuffers_iOS.xcodeproj \
-    -scheme ProtocolBuffers \
-    -sdk iphonesimulator \
-    build-tests
-  IOS_DESTINATIONS=(
-    "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit
-    "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit
-    "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit
-    "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit
-  )
-  for i in "${IOS_DESTINATIONS[@]}" ; do
-    internal_xctool_debug_and_release \
-      -project objectivec/ProtocolBuffers_iOS.xcodeproj \
-      -scheme ProtocolBuffers \
-      -sdk iphonesimulator \
-      -destination "${i}" \
-      run-tests
-  done
+  # Reused the build script that takes care of configuring and ensuring things
+  # are up to date.  The OS X test runs the objc conformance test, so skip it
+  # here.
+  # Note: travis has xctool installed, and we've looked at using it in the past
+  # but it has ended up proving unreliable (bugs), an they are removing build
+  # support in favor of xcbuild (or just xcodebuild).
+  objectivec/DevTools/full_mac_build.sh \
+      --core-only --skip-xcode-osx --skip-objc-conformance "$@"
+}
+
+build_objectivec_ios_debug() {
+  build_objectivec_ios --skip-xcode-release
+}
+
+build_objectivec_ios_release() {
+  build_objectivec_ios --skip-xcode-debug
 }

 build_objectivec_osx() {
-  internal_objectivec_common
-  internal_xctool_debug_and_release \
-    -project objectivec/ProtocolBuffers_OSX.xcodeproj \
-    -scheme ProtocolBuffers \
-    -destination "platform=OS X,arch=x86_64" \
-    test
-  cd conformance && make test_objc && cd ..
+  # Reused the build script that takes care of configuring and ensuring things
+  # are up to date.
+  objectivec/DevTools/full_mac_build.sh \
+      --core-only --skip-xcode-ios
 }

 build_python() {
@ -330,6 +301,8 @@ Usage: $0 { cpp |
            javanano_jdk7 |
            javanano_oracle7 |
            objectivec_ios |
+            objectivec_ios_debug |
+            objectivec_ios_release |
            objectivec_osx |
            python |
            python_cpp |