v8/test/cctest/cctest.status
machenbach 03f5124822 [test] Enable test status filtering by variant
This adds the possibility to address test cases in the
status file with the variant under which the test is running.
This is only allowed in top-level sections.

Example:
[{
  'test-case': [PASS, SLOW],
}]

['variant == foo', {
  'test-case': [FAIL],
}]

The test case "test-case" is marked as slow in all variants.
Additionally, in variant foo, it'll be expected to fail.

This CL also exemplifies the new feature with test cases
running under the ignition_turbofan variant. The
corresponding legacy flag is deprecated.

BUG=v8:5238

Review-Url: https://codereview.chromium.org/2203013002
Cr-Commit-Position: refs/heads/master@{#38342}
2016-08-04 14:42:16 +00:00

463 lines
20 KiB
Plaintext

# Copyright 2011 the V8 project authors. All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are
# met:
#
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above
# copyright notice, this list of conditions and the following
# disclaimer in the documentation and/or other materials provided
# with the distribution.
# * Neither the name of Google Inc. nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
[
[ALWAYS, {
# All tests prefixed with 'Bug' are expected to fail.
'test-api/Bug*': [FAIL],
'test-serialize/Bug*': [FAIL],
##############################################################################
# BUG(382): Weird test. Can't guarantee that it never times out.
'test-api/ApplyInterruption': [PASS, TIMEOUT],
# These tests always fail. They are here to test test.py. If
# they don't fail then test.py has failed.
'test-serialize/TestThatAlwaysFails': [FAIL],
'test-api/SealHandleScope': [FAIL],
# This test always fails. It tests that LiveEdit causes abort when turned off.
'test-debug/LiveEditDisabled': [FAIL],
# This test always fails. It tests that DisallowJavascriptExecutionScope
# works as intended.
'test-api/DisallowJavascriptExecutionScope': [FAIL],
# TODO(gc): Temporarily disabled in the GC branch.
'test-log/EquivalenceOfLoggingAndTraversal': [PASS, FAIL],
# We do not yet shrink weak maps after they have been emptied by the GC
'test-weakmaps/Shrinking': [FAIL],
'test-weaksets/WeakSet_Shrinking': [FAIL],
# Boot up memory use is bloated in debug mode.
'test-mark-compact/BootUpMemoryUse': [PASS, PASS, ['mode == debug', FAIL]],
# This tests only that the preparser and parser agree, so there is no point in
# running several variants. Note that this still takes ages, because there
# are actually 13 * 38 * 5 * 128 = 316160 individual tests hidden here.
'test-parsing/ParserSync': [PASS, NO_VARIANTS],
# This tests only the type system, no point in running several variants.
'test-hydrogen-types/*': [PASS, NO_VARIANTS],
'test-types/*': [PASS, NO_VARIANTS],
# This tests API threading, no point in running several variants.
'test-api/Threading*': [PASS, NO_VARIANTS],
# BUG(2999). The cpu profiler tests are notoriously flaky.
'test-cpu-profiler/CpuProfileDeepStack': [SKIP],
'test-cpu-profiler/CollectCpuProfile': [SKIP],
'test-cpu-profiler/HotDeoptNoFrameEntry': [SKIP],
'test-cpu-profiler/SampleWhenFrameIsNotSetup': [SKIP],
# BUG(v8:4358). Hangs flakily.
'test-debug/ProcessDebugMessagesThreaded': [SKIP],
# BUG(2340). Preprocessing stack traces is disabled at the moment.
'test-heap/PreprocessStackTrace': [FAIL],
# BUG(4333). Function name inferrer does not work for ES6 clases.
'test-func-name-inference/UpperCaseClass': [FAIL],
'test-func-name-inference/LowerCaseClass': [FAIL],
##############################################################################
# TurboFan compiler failures.
# Some tests are just too slow to run for now.
'test-heap/IncrementalMarkingStepMakesBigProgressWithLargeObjects': [PASS, NO_VARIANTS],
'test-heap-profiler/ManyLocalsInSharedContext': [PASS, NO_VARIANTS],
'test-serialize/CodeSerializerLargeCodeObject': [PASS, NO_VARIANTS],
'test-debug/ThreadedDebugging': [PASS, NO_VARIANTS],
# BUG(3742).
'test-mark-compact/MarkCompactCollector': [PASS, ['arch==arm', NO_VARIANTS]],
# TODO(bmeurer): TurboFan embeds strong references to all kinds of objects
# via deoptimization data (Crankshaft also does this, but lack proper test
# coverage).
'test-heap/ObjectsInOptimizedCodeAreWeak': [PASS, NO_VARIANTS],
# TurboFan cpu profiler result is different.
'test-cpu-profiler/CollectDeoptEvents': [PASS, NO_VARIANTS],
'test-cpu-profiler/DeoptAtFirstLevelInlinedSource': [PASS, NO_VARIANTS],
'test-cpu-profiler/DeoptAtSecondLevelInlinedSource': [PASS, NO_VARIANTS],
############################################################################
# Slow tests.
'test-api/Threading1': [PASS, ['mode == debug', SLOW]],
'test-api/Threading2': [PASS, ['mode == debug', SLOW]],
'test-api/Threading3': [PASS, ['mode == debug', SLOW]],
'test-api/Threading4': [PASS, ['mode == debug', SLOW]],
'test-debug/CallFunctionInDebugger': [PASS, ['mode == debug', SLOW]],
'test-strings/StringOOM*': [PASS, ['mode == debug', SKIP]],
'test-serialize/CustomSnapshotDataBlobImmortalImmovableRoots': [PASS, ['mode == debug', SKIP]],
############################################################################
# Ignition
# TODO(rmcilroy,4680): Related to lack of code flushing. Check failed: !function->shared()->is_compiled() || function->IsOptimized().
'test-heap/TestCodeFlushingPreAged': [PASS, NO_IGNITION],
'test-heap/TestCodeFlushingIncrementalScavenge': [PASS, NO_IGNITION],
'test-heap/TestCodeFlushing': [PASS, NO_IGNITION],
'test-heap/TestCodeFlushingIncremental': [PASS, NO_IGNITION],
'test-heap/TestCodeFlushingIncrementalAbort': [PASS, ['mode == debug or dcheck_always_on == True', NO_IGNITION]],
# TODO(mythria,4780): Related to type feedback support for Array function.
'test-feedback-vector/VectorCallFeedbackForArray': [PASS, NO_IGNITION],
# TODO(mythria,4780): Related to type feedback support for constructor.
'test-feedback-vector/VectorConstructCounts': [PASS, NO_IGNITION],
'test-heap/WeakFunctionInConstructor': [PASS, NO_IGNITION],
'test-heap/IncrementalMarkingClearsMonomorphicConstructor': [PASS, NO_IGNITION],
'test-heap/IncrementalMarkingPreservesMonomorphicConstructor': [PASS, NO_IGNITION],
# TODO(mythria,4680): Lack of code-ageing in interpreter.
'test-heap/Regress169209': [PASS, NO_IGNITION],
# TODO(mythria,4680): Lack of code-ageing and/or lack of compilation cache
# in interpreter.
'test-heap/CompilationCacheCachingBehavior': [PASS, NO_IGNITION],
# BUG(rmcilroy,4680): Function is optimized without type feedback and so immediately deopts again, causing check failure in the test.
'test-heap/ResetSharedFunctionInfoCountersDuringIncrementalMarking': [PASS, NO_IGNITION],
'test-heap/ResetSharedFunctionInfoCountersDuringMarkSweep': [PASS, NO_IGNITION],
# BUG(4680): Missing type feedback makes optimistic optimizations fail.
'test-cpu-profiler/DeoptUntrackedFunction': [PASS, NO_IGNITION],
# BUG(4680): Ignition doesn't support allocation sites currently.
'test-heap/EnsureAllocationSiteDependentCodesProcessed': [PASS, NO_IGNITION],
'test-heap/OptimizedPretenuringAllocationFolding': [PASS, NO_IGNITION],
'test-heap/OptimizedPretenuringdoubleArrayLiterals': [PASS, NO_IGNITION],
'test-heap/OptimizedPretenuringNestedDoubleLiterals': [PASS, NO_IGNITION],
'test-heap/OptimizedPretenuringNestedMixedArrayLiterals': [PASS, NO_IGNITION],
'test-heap/OptimizedPretenuringNestedObjectLiterals': [PASS, NO_IGNITION],
'test-heap/OptimizedPretenuringObjectArrayLiterals': [PASS, NO_IGNITION],
# BUG(4751). Flaky with ignition.
'test-cpu-profiler/JsNativeJsSample': [PASS, NO_IGNITION],
# TODO(ignition): Fails due to missing type info when optimizing from bytecode
# with crankshaft.
'test-cpu-profiler/TickLinesOptimized': [PASS, NO_IGNITION],
}], # ALWAYS
##############################################################################
['arch == arm64', {
'test-cpu-profiler/CollectDeoptEvents': [PASS, FAIL],
'test-api/Bug618': [PASS],
# BUG(v8:3385).
'test-serialize/StartupSerializerOnceRunScript': [PASS, FAIL],
'test-serialize/StartupSerializerTwiceRunScript': [PASS, FAIL],
# BUG(v8:3154).
'test-heap/ReleaseOverReservedPages': [PASS, FAIL],
# BUG(v8:3155).
'test-strings/OneByteArrayJoin': [PASS, ['mode == debug', FAIL]],
# BUG(v8:3247).
'test-mark-compact/NoPromotion': [SKIP],
# BUG(v8:3446).
'test-mark-compact/Promotion': [PASS, FAIL],
# BUG(v8:3434).
' test-api/LoadICFastApi_DirectCall_GCMoveStubWithProfiler': [SKIP],
}], # 'arch == arm64'
['arch == arm64 and simulator_run == True', {
# Pass but take too long with the simulator.
'test-api/ExternalArrays': [PASS, TIMEOUT],
'test-api/Threading1': [SKIP],
'test-api/Threading2': [SKIP],
}], # 'arch == arm64 and simulator_run == True'
['arch == arm64 and mode == debug and simulator_run == True', {
# Pass but take too long with the simulator in debug mode.
'test-api/ExternalDoubleArray': [SKIP],
'test-api/ExternalFloat32Array': [SKIP],
'test-api/ExternalFloat64Array': [SKIP],
'test-api/ExternalFloatArray': [SKIP],
'test-api/Float32Array': [SKIP],
'test-api/Float64Array': [SKIP],
}], # 'arch == arm64 and mode == debug and simulator_run == True'
##############################################################################
['asan == True', {
# Skip tests not suitable for ASAN.
'test-assembler-x64/AssemblerX64XchglOperations': [SKIP],
}], # 'asan == True'
##############################################################################
['msan == True', {
# ICU upstream issues.
'test-strings/CountBreakIterator': [SKIP],
# Slow tests.
'test-api/Threading1': [PASS, SLOW],
'test-api/Threading2': [PASS, SLOW],
'test-api/Threading3': [PASS, SLOW],
'test-api/Threading4': [PASS, SLOW],
}], # 'msan == True'
##############################################################################
['no_snap == True', {
# Slow tests.
'test-api/ExternalWrap': [PASS, SLOW],
'test-api/FastReturnValues*': [PASS, SLOW],
'test-decls/CrossScriptReferences_Simple2': [PASS, SLOW],
}], # 'no_snap == True'
##############################################################################
# TODO(machenbach): Fix application of '*'. Nosnap windows needs a separate
# section to not overwrite the expectations for TestThatAlwaysFails.
['no_snap == True and system == windows', {
# Windows doesn't support nosnap mode.
'test-serialize/*': [SKIP],
}], # 'no_snap == True and system == windows'
##############################################################################
['system == windows', {
# BUG(3331). Fails on windows.
'test-heap/NoWeakHashTableLeakWithIncrementalMarking': [SKIP],
# BUG(v8:4573).
'test-api/MultipleIsolatesOnIndividualThreads': [PASS, NO_VARIANTS],
# BUG(v8:4642).
'test-lockers/LockAndUnlockDifferentIsolates': [PASS, NO_VARIANTS],
# BUG(v8:2999,v8:4751).
'test-cpu-profiler/CollectDeoptEvents': [SKIP],
'test-cpu-profiler/FunctionCallSample': [SKIP],
'test-cpu-profiler/JsNativeJsRuntimeJsSample': [SKIP],
'test-cpu-profiler/JsNativeJsRuntimeJsSampleMultiple': [SKIP],
'test-cpu-profiler/JsNativeJsSample': [SKIP],
'test-cpu-profiler/JsNative1JsNative2JsSample': [SKIP],
}], # 'system == windows'
##############################################################################
['arch == arm', {
'test-cpu-profiler/CollectDeoptEvents': [PASS, FAIL],
# BUG(355): Test crashes on ARM.
'test-log/ProfLazyMode': [SKIP],
# BUG(1075): Unresolved crashes.
'test-serialize/StartupSerializerOnce': [SKIP],
'test-serialize/StartupSerializerTwice': [SKIP],
'test-serialize/StartupSerializerOnceRunScript': [SKIP],
'test-serialize/StartupSerializerTwiceRunScript': [SKIP],
############################################################################
# Slow tests.
'test-api/Threading1': [PASS, SLOW],
'test-api/Threading2': [PASS, SLOW],
'test-api/Threading3': [PASS, SLOW],
'test-api/Threading4': [PASS, SLOW],
}], # 'arch == arm'
##############################################################################
['arch == mipsel or arch == mips', {
'test-cpu-profiler/CollectDeoptEvents': [PASS, FAIL],
# TODO(mips-team): Improve code-size on large RegExp's.
'test-heap/TestSizeOfRegExpCode': [SKIP],
# BUG(1075): Unresolved crashes on MIPS also.
'test-serialize/StartupSerializerOnce': [SKIP],
'test-serialize/StartupSerializerTwice': [SKIP],
'test-serialize/StartupSerializerOnceRunScript': [SKIP],
'test-serialize/StartupSerializerTwiceRunScript': [SKIP],
}], # 'arch == mipsel or arch == mips'
##############################################################################
['arch == mips', {
# Too slow with TF.
'test-api/ExternalArrays': [PASS, NO_VARIANTS],
# TODO(mips-team): Currently fails on mips board.
'test-parsing/TooManyArguments': [SKIP],
'test-api/Threading3': [SKIP],
}], # 'arch == mips'
##############################################################################
['arch == mips64el or arch == mips64', {
'test-cpu-profiler/CollectDeoptEvents': [PASS, FAIL],
# BUG(v8:3154).
'test-heap/ReleaseOverReservedPages': [PASS, FAIL],
# TODO(mips-team): Improve code-size on large RegExp's.
'test-heap/TestSizeOfRegExpCode': [SKIP],
# BUG(1075): Unresolved crashes on MIPS also.
'test-serialize/StartupSerializerOnce': [SKIP],
'test-serialize/StartupSerializerTwice': [SKIP],
'test-serialize/StartupSerializerOnceRunScript': [SKIP],
'test-serialize/StartupSerializerTwiceRunScript': [SKIP],
}], # 'arch == mips64el or arch == mips64'
##############################################################################
['arch == x87', {
'test-run-machops/RunFloat64InsertLowWord32': [SKIP],
'test-run-native-calls/MixedParams_0': [SKIP],
'test-run-native-calls/MixedParams_1': [SKIP],
'test-run-native-calls/MixedParams_2': [SKIP],
'test-run-native-calls/MixedParams_3': [SKIP],
'test-run-machops/RunFloat64MulAndFloat64Add1': [SKIP],
'test-run-machops/RunFloat64MulAndFloat64Add2': [SKIP],
'test-run-machops/RunFloat64MulAndFloat64Sub1': [SKIP],
'test-run-machops/RunFloat64MulAndFloat64Sub2': [SKIP],
'test-run-machops/RunFloat64Sin': [SKIP],
'test-run-machops/RunFloat64Cos': [SKIP],
'test-run-machops/RunFloat64Expm1': [SKIP],
'test-run-machops/RunFloat64Tan': [SKIP],
'test-cpu-profiler/Inlining': [SKIP],
'test-gap-resolver/FuzzResolver': [SKIP],
}], # 'arch == x87'
##############################################################################
['arch == android_arm or arch == android_ia32', {
# Tests crash as there is no /tmp directory in Android.
'test-log/LogAccessorCallbacks': [SKIP],
'test-log/LogCallbacks': [SKIP],
'test-log/ProfLazyMode': [SKIP],
}], # 'arch == android_arm or arch == android_ia32'
##############################################################################
['system == aix and arch == ppc64', {
# BUG 2857
'test-log/EquivalenceOfLoggingAndTraversal' : [SKIP],
}], # 'system == aix and arch == ppc64'
##############################################################################
['system == aix or (arch == ppc64 and byteorder == big)', {
# TODO(ppc): Fix for platforms with function desciptors.
'test-run-machops/RunComputedCodeObject' : [SKIP],
'test-run-wasm-module/Run_WasmModule_Return114' : [SKIP],
'test-run-wasm-module/Run_WasmModule_CallAdd' : [SKIP],
'test-run-wasm-module/Run_WasmModule_CallMain_recursive' : [SKIP],
}], # 'system == aix or (arch == ppc64 and byteorder == big)'
##############################################################################
['arch == ppc and simulator_run == True or arch == ppc64 and simulator_run == True', {
# Pass but take too long with the simulator.
'test-api/Threading1': [PASS, SLOW],
'test-api/Threading2': [PASS, SLOW],
'test-api/ExternalArrays': [PASS, SLOW],
}], # 'arch == ppc64 and simulator_run == True'
##############################################################################
['variant == ignition_turbofan', {
# TODO(5251): Inlining is currently disabled for the BytecodeGraphBuilder.
'test-run-inlining/InlineLoopGuardedTwice': [FAIL],
'test-run-inlining/InlineSurplusArgumentsDeopt': [FAIL],
'test-run-inlining/InlineTwice': [FAIL],
'test-run-inlining/InlineSurplusArgumentsObject': [FAIL],
'test-run-inlining/InlineTwiceDependentDiamond': [FAIL],
'test-run-inlining/InlineWithArguments': [FAIL],
'test-run-inlining/InlineLoopUnguardedTwice': [FAIL],
'test-run-inlining/InlineOmitArgumentsObject': [FAIL],
'test-run-inlining/InlineLoopUnguardedOnce': [FAIL],
'test-run-inlining/InlineOmitArgumentsDeopt': [FAIL],
'test-run-inlining/InlineTwiceDependentDiamondDifferent': [FAIL],
'test-run-inlining/SimpleInliningContext': [FAIL],
'test-run-inlining/InlineMutuallyRecursive': [FAIL],
'test-run-inlining/InlineLoopGuardedEmpty': [FAIL],
'test-run-inlining/InlineLoopGuardedOnce': [FAIL],
'test-run-inlining/InlineOmitArguments': [FAIL],
'test-run-inlining/SimpleInlining': [FAIL],
'test-run-inlining/InlineLoopUnguardedEmpty': [FAIL],
'test-run-inlining/InlineNestedBuiltin': [FAIL],
'test-run-inlining/InlineSurplusArguments': [FAIL],
'test-run-inlining/InlineBuiltin': [FAIL],
'test-run-inlining/InlineTwiceDependent': [FAIL],
'test-run-inlining/SimpleInliningContextDeopt': [FAIL],
# TODO(rmcilroy,4766): Requires BytecodeGraphBuilder to track source position
# on nodes (behind --turbo_source_positions flag).
'test-cpu-profiler/TickLinesOptimized': [FAIL],
# TODO(rmcilroy,4680): Related to lack of code flushing. Check failed: !function->shared()->is_compiled() || function->IsOptimized().
'test-heap/TestCodeFlushingPreAged': [FAIL],
'test-heap/TestCodeFlushingIncrementalScavenge': [FAIL],
'test-heap/TestCodeFlushing': [FAIL],
'test-heap/TestCodeFlushingIncremental': [FAIL],
'test-heap/TestCodeFlushingIncrementalAbort': [PASS, ['mode == debug or dcheck_always_on == True', FAIL]],
# TODO(mythria,4780): Related to type feedback support for Array function.
'test-feedback-vector/VectorCallFeedbackForArray': [FAIL],
# TODO(mythria,4780): Related to type feedback support for constructor.
'test-feedback-vector/VectorConstructCounts': [FAIL],
'test-heap/WeakFunctionInConstructor': [FAIL],
'test-heap/IncrementalMarkingClearsMonomorphicConstructor': [FAIL],
'test-heap/IncrementalMarkingPreservesMonomorphicConstructor': [FAIL],
# TODO(mythria,4680): Lack of code-ageing in interpreter.
'test-heap/Regress169209': [FAIL],
# TODO(mythria,4680): Lack of code-ageing and/or lack of compilation cache
# in interpreter.
'test-heap/CompilationCacheCachingBehavior': [FAIL],
# BUG(4680): Missing type feedback makes optimistic optimizations fail.
'test-cpu-profiler/CollectDeoptEvents': [FAIL],
'test-cpu-profiler/DeoptUntrackedFunction': [FAIL],
# BUG(4680): Ignition doesn't support allocation sites currently.
'test-heap/EnsureAllocationSiteDependentCodesProcessed': [FAIL],
'test-heap/OptimizedPretenuringAllocationFolding': [FAIL],
'test-heap/OptimizedPretenuringdoubleArrayLiterals': [FAIL],
'test-heap/OptimizedPretenuringNestedDoubleLiterals': [FAIL],
'test-heap/OptimizedPretenuringNestedMixedArrayLiterals': [FAIL],
'test-heap/OptimizedPretenuringNestedObjectLiterals': [FAIL],
'test-heap/OptimizedPretenuringObjectArrayLiterals': [FAIL],
# BUG(4751). Flaky with Ignition.
'test-cpu-profiler/JsNativeJsSample': [SKIP],
}], # variant == ignition_turbofan
]