AuroraMiddleware/v8
1
0
Fork 0
Code Issues 2 Pull Requests Releases Wiki Activity

[testrunner] show the variant type along with the test name

Browse Source 
The test name alone isn't informative alone.

This CL prints the variant name during the progress reporting.

tested by running test262 with all variants

R=machenbach@chromium.org
CC=yangguo@chromium.org,sergiyb@chromium.org

Bug: v8:8728
Change-Id: I3d7d1fc6eacef3712d87404c7276fd5523bb445a
Reviewed-on: https://chromium-review.googlesource.com/c/1458156
Commit-Queue: Tamer Tas <tmrts@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
Cr-Commit-Position: refs/heads/master@{#59437}
This commit is contained in:
Tamer Tas 2019-02-07 14:03:13 +01:00 committed by Commit Bot
parent 3d3872cba8
commit 5aee0e1710
4 changed files with 128 additions and 126 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
tools/testrunner/testproc/progress.py
View File

@ -113,7 +113,9 @@ class VerboseProgressIndicator(SimpleProgressIndicator):
outcome = 'FAIL'
else:
outcome = 'pass'
self._print('Done running %s: %s' % (test, outcome))
self._print('Done running %s %s: %s' % (
test, test.variant or 'default', outcome))
def _on_heartbeat(self):
if time.time() - self._last_printed_time > 30:

17
tools/unittests/run_tests_test.py
View File

@ -173,7 +173,7 @@ class SystemTest(unittest.TestCase):
'sweet/bananas',
'sweet/raspberries',
)
self.assertIn('Done running sweet/bananas: pass', result.stdout, result)
self.assertIn('Done running sweet/bananas default: pass', result.stdout, result)
# TODO(majeski): Implement for test processors
# self.assertIn('Total time:', result.stderr, result)
# self.assertIn('sweet/bananas', result.stderr, result)
@ -233,7 +233,7 @@ class SystemTest(unittest.TestCase):
'sweet/strawberries',
infra_staging=False,
)
self.assertIn('Done running sweet/strawberries: FAIL', result.stdout, result)
self.assertIn('Done running sweet/strawberries default: FAIL', result.stdout, result)
self.assertEqual(1, result.returncode, result)
def check_cleaned_json_output(
@ -278,7 +278,7 @@ class SystemTest(unittest.TestCase):
'sweet/strawberries',
infra_staging=False,
)
self.assertIn('Done running sweet/strawberries: FAIL', result.stdout, result)
self.assertIn('Done running sweet/strawberries default: FAIL', result.stdout, result)
# With test processors we don't count reruns as separated failures.
# TODO(majeski): fix it?
self.assertIn('1 tests failed', result.stdout, result)
@ -308,7 +308,7 @@ class SystemTest(unittest.TestCase):
infra_staging=False,
)
self.assertIn(
'Done running sweet/bananaflakes: pass', result.stdout, result)
'Done running sweet/bananaflakes default: pass', result.stdout, result)
self.assertIn('All tests succeeded', result.stdout, result)
self.assertEqual(0, result.returncode, result)
self.maxDiff = None
@ -498,7 +498,8 @@ class SystemTest(unittest.TestCase):
infra_staging=False,
)
self.assertIn('1 tests ran', result.stdout, result)
self.assertIn('Done running sweet/bananas: FAIL', result.stdout, result)
self.assertIn(
'Done running sweet/bananas default: FAIL', result.stdout, result)
self.assertIn('Test had no allocation output', result.stdout, result)
self.assertIn('--predictable --verify_predictable', result.stdout, result)
self.assertEqual(1, result.returncode, result)
@ -634,10 +635,10 @@ class SystemTest(unittest.TestCase):
'sweet/blackberries', # FAIL
'sweet/raspberries', # should not run
)
self.assertIn('sweet/mangoes: pass', result.stdout, result)
self.assertIn('sweet/strawberries: FAIL', result.stdout, result)
self.assertIn('sweet/mangoes default: pass', result.stdout, result)
self.assertIn('sweet/strawberries default: FAIL', result.stdout, result)
self.assertIn('Too many failures, exiting...', result.stdout, result)
self.assertIn('sweet/blackberries: FAIL', result.stdout, result)
self.assertIn('sweet/blackberries default: FAIL', result.stdout, result)
self.assertNotIn('Done running sweet/raspberries', result.stdout, result)
self.assertIn('2 tests failed', result.stdout, result)
self.assertIn('3 tests ran', result.stdout, result)

139
tools/unittests/testdata/expected_test_results1.json vendored
View File

@ -1,113 +1,112 @@
{
"arch": "x64",
"duration_mean": 1,
"mode": "release",
"arch": "x64",
"duration_mean": 1,
"mode": "release",
"results": [
{
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
"duration": 1,
"exit_code": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
"duration": 1,
"exit_code": 1,
"expected": [
"PASS"
],
],
"flags": [
"--test",
"strawberries",
"--random-seed=123",
"strawberries",
"--random-seed=123",
"--nohard-abort"
],
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 1,
"stderr": "",
"stdout": "--test strawberries --random-seed=123 --nohard-abort\n",
"target_name": "d8_mocked.py",
],
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 1,
"stderr": "",
"stdout": "--test strawberries --random-seed=123 --nohard-abort\n",
"target_name": "d8_mocked.py",
"variant": "default"
},
},
{
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
"duration": 1,
"exit_code": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
"duration": 1,
"exit_code": 1,
"expected": [
"PASS"
],
],
"flags": [
"--test",
"strawberries",
"--random-seed=123",
"strawberries",
"--random-seed=123",
"--nohard-abort"
],
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 2,
"stderr": "",
"stdout": "--test strawberries --random-seed=123 --nohard-abort\n",
"target_name": "d8_mocked.py",
],
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 2,
"stderr": "",
"stdout": "--test strawberries --random-seed=123 --nohard-abort\n",
"target_name": "d8_mocked.py",
"variant": "default"
},
},
{
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
"duration": 1,
"exit_code": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
"duration": 1,
"exit_code": 1,
"expected": [
"PASS"
],
],
"flags": [
"--test",
"strawberries",
"--random-seed=123",
"strawberries",
"--random-seed=123",
"--nohard-abort"
],
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 3,
"stderr": "",
"stdout": "--test strawberries --random-seed=123 --nohard-abort\n",
"target_name": "d8_mocked.py",
],
"name": "sweet/strawberries",
"random_seed": 123,
"result": "FAIL",
"run": 3,
"stderr": "",
"stdout": "--test strawberries --random-seed=123 --nohard-abort\n",
"target_name": "d8_mocked.py",
"variant": "default"
}
],
],
"slowest_tests": [
{
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
"duration": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
"duration": 1,
"flags": [
"--test",
"strawberries",
"--random-seed=123",
"strawberries",
"--random-seed=123",
"--nohard-abort"
],
"marked_slow": true,
],
"marked_slow": true,
"name": "sweet/strawberries"
},
},
{
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
"duration": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
"duration": 1,
"flags": [
"--test",
"strawberries",
"--random-seed=123",
"strawberries",
"--random-seed=123",
"--nohard-abort"
],
"marked_slow": true,
],
"marked_slow": true,
"name": "sweet/strawberries"
},
},
{
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
"duration": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py --test strawberries --random-seed=123 --nohard-abort",
"duration": 1,
"flags": [
"--test",
"strawberries",
"--random-seed=123",
"strawberries",
"--random-seed=123",
"--nohard-abort"
],
"marked_slow": true,
],
"marked_slow": true,
"name": "sweet/strawberries"
}
],
],
"test_total": 3
}

94
tools/unittests/testdata/expected_test_results2.json vendored
View File

@ -1,74 +1,74 @@
{
"arch": "x64",
"duration_mean": 1,
"mode": "release",
"arch": "x64",
"duration_mean": 1,
"mode": "release",
"results": [
{
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort",
"duration": 1,
"exit_code": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort",
"duration": 1,
"exit_code": 1,
"expected": [
"PASS"
],
],
"flags": [
"bananaflakes",
"--random-seed=123",
"bananaflakes",
"--random-seed=123",
"--nohard-abort"
],
"name": "sweet/bananaflakes",
"random_seed": 123,
"result": "FAIL",
"run": 1,
"stderr": "",
"stdout": "bananaflakes --random-seed=123 --nohard-abort\n",
"target_name": "d8_mocked.py",
],
"name": "sweet/bananaflakes",
"random_seed": 123,
"result": "FAIL",
"run": 1,
"stderr": "",
"stdout": "bananaflakes --random-seed=123 --nohard-abort\n",
"target_name": "d8_mocked.py",
"variant": "default"
},
},
{
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort",
"duration": 1,
"exit_code": 0,
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort",
"duration": 1,
"exit_code": 0,
"expected": [
"PASS"
],
],
"flags": [
"bananaflakes",
"--random-seed=123",
"bananaflakes",
"--random-seed=123",
"--nohard-abort"
],
"name": "sweet/bananaflakes",
"random_seed": 123,
"result": "PASS",
"run": 2,
"stderr": "",
"stdout": "bananaflakes --random-seed=123 --nohard-abort\n",
"target_name": "d8_mocked.py",
],
"name": "sweet/bananaflakes",
"random_seed": 123,
"result": "PASS",
"run": 2,
"stderr": "",
"stdout": "bananaflakes --random-seed=123 --nohard-abort\n",
"target_name": "d8_mocked.py",
"variant": "default"
}
],
],
"slowest_tests": [
{
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort",
"duration": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort",
"duration": 1,
"flags": [
"bananaflakes",
"--random-seed=123",
"bananaflakes",
"--random-seed=123",
"--nohard-abort"
],
"marked_slow": false,
],
"marked_slow": false,
"name": "sweet/bananaflakes"
},
},
{
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort",
"duration": 1,
"command": "/usr/bin/python out/Release/d8_mocked.py bananaflakes --random-seed=123 --nohard-abort",
"duration": 1,
"flags": [
"bananaflakes",
"--random-seed=123",
"bananaflakes",
"--random-seed=123",
"--nohard-abort"
],
"marked_slow": false,
],
"marked_slow": false,
"name": "sweet/bananaflakes"
}
],
],
"test_total": 2
}