From 368a2f4ceea7611a1c4435a056cf7a44bdcdb16d Mon Sep 17 00:00:00 2001 From: Thomas Van Lenten Date: Tue, 24 May 2016 11:27:33 -0400 Subject: [PATCH] Automated testing tweaks for ObjC - Move the ObjC tests into the list and exclude them on linux, this will change where in the order they start, since they are longer, it will have other things run in parallel instead of them ending up last and taking the longest. - Switch to the Xcode 7.3 image. - Drop the use of xctool and stream line things through the full_mac_build.sh script. This means we end up with only one build script instead of two. - Tweaks to the mac build script: - Make iOS Xcode version support explicit - Support Debug/Release only building - Change the OS X min parallel count to 2 to better deal with VMs. - Split the travis ios tests into the two Xcode Configurations as the logs are choking travis. --- .travis.yml | 33 ++++++------ objectivec/DevTools/full_mac_build.sh | 73 +++++++++++++++++++-------- tests.sh | 71 ++++++++------------------ 3 files changed, 91 insertions(+), 86 deletions(-) diff --git a/.travis.yml b/.travis.yml index bcf3851bc..86451edda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -8,7 +8,7 @@ os: - linux - osx # The Objective C build needs Xcode 7.0 or later. -osx_image: xcode7.2 +osx_image: xcode7.3 script: - ./tests.sh $CONFIG env: @@ -23,6 +23,11 @@ env: - CONFIG=javanano_jdk7 - CONFIG=javanano_oracle7 - CONFIG=javascript + # iOS build log was starting to choke travis UI, so split to cover the + # Xcode Debug and Release Configurations independently. + - CONFIG=objectivec_ios_debug + - CONFIG=objectivec_ios_release + - CONFIG=objectivec_osx - CONFIG=python - CONFIG=python_cpp - CONFIG=ruby19 @@ -54,12 +59,13 @@ matrix: # which doesn't work on OS X. - os: osx env: CONFIG=golang - # Add into the matrix OS X tests of Objective C (needs Xcode, so it won't - # work on other platforms). These are split so it doesn't take as long to run. - include: - - os: osx - env: CONFIG=objectivec_ios - - os: osx + # OS X/iOS tests of Objective C (needs Xcode, so it won't work on other + # platforms). + - os: linux + env: CONFIG=objectivec_ios_debug + - os: linux + env: CONFIG=objectivec_ios_release + - os: linux env: CONFIG=objectivec_osx allow_failures: # These currently do not work on OS X but are being worked on by @haberman. @@ -71,14 +77,11 @@ matrix: # we moved to an OS X image that is 10.11. - os: osx env: CONFIG=python_cpp - # xctool 0.2.8 seems to have a bug where it randomly kills tests saying - # they failed. - # https://github.com/facebook/xctool/issues/619 - # https://github.com/google/protobuf/issues/1232 - # travis updated their images to include 0.2.8: - # https://blog.travis-ci.com/2016-03-23-xcode-image-updates - # Mark the iOS test as flakey so these failures don't turn things red. + # Mark the iOS test as flakey as xcodebuild some times fails to start the + # iOS Simulator. - os: osx - env: CONFIG=objectivec_ios + env: CONFIG=objectivec_ios_debug + - os: osx + env: CONFIG=objectivec_ios_release notifications: email: false diff --git a/objectivec/DevTools/full_mac_build.sh b/objectivec/DevTools/full_mac_build.sh index ff51d9f01..89240ee02 100755 --- a/objectivec/DevTools/full_mac_build.sh +++ b/objectivec/DevTools/full_mac_build.sh @@ -37,6 +37,10 @@ OPTIONS: Skip the invoke of Xcode to test the runtime on both iOS and OS X. --skip-xcode-ios Skip the invoke of Xcode to test the runtime on iOS. + --skip-xcode-debug + Skip the Xcode Debug configuration. + --skip-xcode-release + Skip the Xcode Release configuration. --skip-xcode-osx Skip the invoke of Xcode to test the runtime on OS X. --skip-objc-conformance @@ -66,8 +70,8 @@ wrapped_make() { } NUM_MAKE_JOBS=$(/usr/sbin/sysctl -n hw.ncpu) -if [[ "${NUM_MAKE_JOBS}" -lt 4 ]] ; then - NUM_MAKE_JOBS=4 +if [[ "${NUM_MAKE_JOBS}" -lt 2 ]] ; then + NUM_MAKE_JOBS=2 fi DO_AUTOGEN=no @@ -76,6 +80,8 @@ REGEN_DESCRIPTORS=no CORE_ONLY=no DO_XCODE_IOS_TESTS=yes DO_XCODE_OSX_TESTS=yes +DO_XCODE_DEBUG=yes +DO_XCODE_RELEASE=yes DO_OBJC_CONFORMANCE_TESTS=yes while [[ $# != 0 ]]; do case "${1}" in @@ -109,6 +115,12 @@ while [[ $# != 0 ]]; do --skip-xcode-osx ) DO_XCODE_OSX_TESTS=no ;; + --skip-xcode-debug ) + DO_XCODE_DEBUG=no + ;; + --skip-xcode-release ) + DO_XCODE_RELEASE=no + ;; --skip-objc-conformance ) DO_OBJC_CONFORMANCE_TESTS=no ;; @@ -151,8 +163,12 @@ if [[ "${DO_CLEAN}" == "yes" ]] ; then -project objectivec/ProtocolBuffers_iOS.xcodeproj -scheme ProtocolBuffers ) - "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Debug clean - "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Release clean + if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then + "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Debug clean + fi + if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then + "${XCODEBUILD_CLEAN_BASE_IOS[@]}" -configuration Release clean + fi fi if [[ "${DO_XCODE_OSX_TESTS}" == "yes" ]] ; then XCODEBUILD_CLEAN_BASE_OSX=( @@ -160,8 +176,12 @@ if [[ "${DO_CLEAN}" == "yes" ]] ; then -project objectivec/ProtocolBuffers_OSX.xcodeproj -scheme ProtocolBuffers ) - "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Debug clean - "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Release clean + if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then + "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Debug clean + fi + if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then + "${XCODEBUILD_CLEAN_BASE_OSX[@]}" -configuration Release clean + fi fi fi @@ -222,6 +242,14 @@ if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then -destination "platform=iOS Simulator,name=iPad Air,OS=9.0" # 64bit ) ;; + 7.2* ) + XCODEBUILD_TEST_BASE_IOS+=( + -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit + -destination "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit + -destination "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit + -destination "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit + ) + ;; 7.3* ) XCODEBUILD_TEST_BASE_IOS+=( -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit @@ -230,23 +258,19 @@ if [[ "${DO_XCODE_IOS_TESTS}" == "yes" ]] ; then -destination "platform=iOS Simulator,name=iPad Air,OS=9.3" # 64bit ) ;; - 7.* ) - XCODEBUILD_TEST_BASE_IOS+=( - -destination "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit - -destination "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit - -destination "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit - -destination "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit - ) - ;; * ) echo "Time to update the simulator targets for Xcode ${XCODE_VERSION}" exit 2 ;; esac - header "Doing Xcode iOS build/tests - Debug" - "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Debug test - header "Doing Xcode iOS build/tests - Release" - "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Release test + if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then + header "Doing Xcode iOS build/tests - Debug" + "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Debug test + fi + if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then + header "Doing Xcode iOS build/tests - Release" + "${XCODEBUILD_TEST_BASE_IOS[@]}" -configuration Release test + fi # Don't leave the simulator in the developer's face. killall "${IOS_SIMULATOR_NAME}" fi @@ -258,13 +282,18 @@ if [[ "${DO_XCODE_OSX_TESTS}" == "yes" ]] ; then # Since the ObjC 2.0 Runtime is required, 32bit OS X isn't supported. -destination "platform=OS X,arch=x86_64" # 64bit ) - header "Doing Xcode OS X build/tests - Debug" - "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Debug test - header "Doing Xcode OS X build/tests - Release" - "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Release test + if [[ "${DO_XCODE_DEBUG}" == "yes" ]] ; then + header "Doing Xcode OS X build/tests - Debug" + "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Debug test + fi + if [[ "${DO_XCODE_RELEASE}" == "yes" ]] ; then + header "Doing Xcode OS X build/tests - Release" + "${XCODEBUILD_TEST_BASE_OSX[@]}" -configuration Release test + fi fi if [[ "${DO_OBJC_CONFORMANCE_TESTS}" == "yes" ]] ; then + header "Running ObjC Conformance Tests" cd conformance wrapped_make -j "${NUM_MAKE_JOBS}" test_objc cd .. diff --git a/tests.sh b/tests.sh index 6a9439a55..cde108fd5 100755 --- a/tests.sh +++ b/tests.sh @@ -197,59 +197,30 @@ internal_install_python_deps() { fi } -internal_objectivec_common () { - # Make sure xctool is up to date. Adapted from - # http://docs.travis-ci.com/user/osx-ci-environment/ - # We don't use a before_install because we test multiple OSes. - brew update - brew outdated xctool || brew upgrade xctool - # Reused the build script that takes care of configuring and ensuring things - # are up to date. Xcode and conformance tests will be directly invoked. - objectivec/DevTools/full_mac_build.sh \ - --core-only --skip-xcode --skip-objc-conformance -} - -internal_xctool_debug_and_release() { - # Always use -reporter plain to avoid escape codes in output (makes travis - # logs easier to read). - xctool -reporter plain -configuration Debug "$@" - xctool -reporter plain -configuration Release "$@" -} - build_objectivec_ios() { - internal_objectivec_common - # https://github.com/facebook/xctool/issues/509 - unlike xcodebuild, xctool - # doesn't support >1 destination, so we have to build first and then run the - # tests one destination at a time. - internal_xctool_debug_and_release \ - -project objectivec/ProtocolBuffers_iOS.xcodeproj \ - -scheme ProtocolBuffers \ - -sdk iphonesimulator \ - build-tests - IOS_DESTINATIONS=( - "platform=iOS Simulator,name=iPhone 4s,OS=8.1" # 32bit - "platform=iOS Simulator,name=iPhone 6,OS=9.2" # 64bit - "platform=iOS Simulator,name=iPad 2,OS=8.1" # 32bit - "platform=iOS Simulator,name=iPad Air,OS=9.2" # 64bit - ) - for i in "${IOS_DESTINATIONS[@]}" ; do - internal_xctool_debug_and_release \ - -project objectivec/ProtocolBuffers_iOS.xcodeproj \ - -scheme ProtocolBuffers \ - -sdk iphonesimulator \ - -destination "${i}" \ - run-tests - done + # Reused the build script that takes care of configuring and ensuring things + # are up to date. The OS X test runs the objc conformance test, so skip it + # here. + # Note: travis has xctool installed, and we've looked at using it in the past + # but it has ended up proving unreliable (bugs), an they are removing build + # support in favor of xcbuild (or just xcodebuild). + objectivec/DevTools/full_mac_build.sh \ + --core-only --skip-xcode-osx --skip-objc-conformance "$@" +} + +build_objectivec_ios_debug() { + build_objectivec_ios --skip-xcode-release +} + +build_objectivec_ios_release() { + build_objectivec_ios --skip-xcode-debug } build_objectivec_osx() { - internal_objectivec_common - internal_xctool_debug_and_release \ - -project objectivec/ProtocolBuffers_OSX.xcodeproj \ - -scheme ProtocolBuffers \ - -destination "platform=OS X,arch=x86_64" \ - test - cd conformance && make test_objc && cd .. + # Reused the build script that takes care of configuring and ensuring things + # are up to date. + objectivec/DevTools/full_mac_build.sh \ + --core-only --skip-xcode-ios } build_python() { @@ -330,6 +301,8 @@ Usage: $0 { cpp | javanano_jdk7 | javanano_oracle7 | objectivec_ios | + objectivec_ios_debug | + objectivec_ios_release | objectivec_osx | python | python_cpp |