From 540484445f4f18d5ff7de8eff394e8ae5c58096e Mon Sep 17 00:00:00 2001 From: Michael Achenbach Date: Sat, 28 Mar 2020 13:45:36 +0100 Subject: [PATCH] [foozzie] Fix more Python3 incompatibilities NOTRY=true TBR=tmrts@chromium.org Bug: chromium:1065624 Change-Id: I6e49c48bb95e10b7fad1ff2c589a2dd459fff562 Reviewed-on: https://chromium-review.googlesource.com/c/v8/v8/+/2124326 Reviewed-by: Michael Achenbach Commit-Queue: Michael Achenbach Cr-Commit-Position: refs/heads/master@{#66900} --- tools/clusterfuzz/v8_commands.py | 8 +++++- tools/clusterfuzz/v8_foozzie.py | 34 ++++++++++++++++++------- tools/clusterfuzz/v8_foozzie_test.py | 38 +++++++++++++++------------- tools/clusterfuzz/v8_suppressions.py | 15 ++++++++--- 4 files changed, 64 insertions(+), 31 deletions(-) diff --git a/tools/clusterfuzz/v8_commands.py b/tools/clusterfuzz/v8_commands.py index 82dbd023f1..e84cd915e3 100644 --- a/tools/clusterfuzz/v8_commands.py +++ b/tools/clusterfuzz/v8_commands.py @@ -12,6 +12,8 @@ from threading import Event, Timer import v8_fuzz_config +PYTHON3 = sys.version_info >= (3, 0) + # List of default flags passed to each d8 run. DEFAULT_FLAGS = [ '--correctness-fuzzer-suppressions', @@ -107,12 +109,16 @@ class Output(object): def Execute(args, cwd, timeout=None): popen_args = [c for c in args if c != ""] + kwargs = {} + if PYTHON3: + kwargs['encoding'] = 'utf-8' try: process = subprocess.Popen( args=popen_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=cwd, + **kwargs ) except Exception as e: sys.stderr.write("Error executing: %s\n" % popen_args) @@ -135,6 +141,6 @@ def Execute(args, cwd, timeout=None): return Output( process.returncode, timeout_event.is_set(), - stdout.decode('utf-8', 'replace').encode('utf-8'), + stdout, process.pid, ) diff --git a/tools/clusterfuzz/v8_foozzie.py b/tools/clusterfuzz/v8_foozzie.py index 733cbe2568..b6638cc377 100755 --- a/tools/clusterfuzz/v8_foozzie.py +++ b/tools/clusterfuzz/v8_foozzie.py @@ -25,6 +25,8 @@ from collections import namedtuple import v8_commands import v8_suppressions +PYTHON3 = sys.version_info >= (3, 0) + CONFIGS = dict( default=[], ignition=[ @@ -283,7 +285,15 @@ def print_difference( first_config_label = '%s,%s' % (options.first.arch, options.first.config) second_config_label = '%s,%s' % (options.second.arch, options.second.config) source_file_text = SOURCE_FILE_TEMPLATE % source if source else '' - print((FAILURE_TEMPLATE % dict( + + if PYTHON3: + first_stdout = first_config_output.stdout + second_stdout = second_config_output.stdout + else: + first_stdout = first_config_output.stdout.decode('utf-8', 'replace') + second_stdout = second_config_output.stdout.decode('utf-8', 'replace') + + text = (FAILURE_TEMPLATE % dict( configs='%s:%s' % (first_config_label, second_config_label), source_file_text=source_file_text, source_key=source_key, @@ -292,13 +302,15 @@ def print_difference( second_config_label=second_config_label, first_config_flags=' '.join(first_command.flags), second_config_flags=' '.join(second_command.flags), - first_config_output= - first_config_output.stdout.decode('utf-8', 'replace'), - second_config_output= - second_config_output.stdout.decode('utf-8', 'replace'), + first_config_output=first_stdout, + second_config_output=second_stdout, source=source, - difference=difference.decode('utf-8', 'replace'), - )).encode('utf-8', 'replace')) + difference=difference, + )) + if PYTHON3: + print(text) + else: + print(text.encode('utf-8', 'replace')) def main(): @@ -312,7 +324,10 @@ def main(): ) # Static bailout based on test case content or metadata. - with open(options.testcase) as f: + kwargs = {} + if PYTHON3: + kwargs['encoding'] = 'utf-8' + with open(options.testcase, 'r', **kwargs) as f: content = f.read() if content_bailout(get_meta_data(content), suppress.ignore_by_metadata): return RETURN_FAIL @@ -354,7 +369,8 @@ def main(): difference, source = suppress.diff(first_config_output, second_config_output) if source: - source_key = hashlib.sha1(source).hexdigest()[:ORIGINAL_SOURCE_HASH_LENGTH] + long_key = hashlib.sha1(source.encode('utf-8')).hexdigest() + source_key = long_key[:ORIGINAL_SOURCE_HASH_LENGTH] else: source_key = ORIGINAL_SOURCE_DEFAULT diff --git a/tools/clusterfuzz/v8_foozzie_test.py b/tools/clusterfuzz/v8_foozzie_test.py index 6da890fb0e..f82afc9e20 100755 --- a/tools/clusterfuzz/v8_foozzie_test.py +++ b/tools/clusterfuzz/v8_foozzie_test.py @@ -19,6 +19,8 @@ try: except NameError: basestring = str +PYTHON3 = sys.version_info >= (3, 0) + BASE_DIR = os.path.dirname(os.path.abspath(__file__)) FOOZZIE = os.path.join(BASE_DIR, 'v8_foozzie.py') TEST_DATA = os.path.join(BASE_DIR, 'testdata') @@ -108,12 +110,12 @@ class UnitTest(unittest.TestCase): one = '' two = '' diff = None, None - self.assertEquals(diff, diff_fun(one, two)) + self.assertEqual(diff, diff_fun(one, two)) one = 'a \n b\nc();' two = 'a \n b\nc();' diff = None, None - self.assertEquals(diff, diff_fun(one, two)) + self.assertEqual(diff, diff_fun(one, two)) # Ignore line before caret, caret position and error message. one = """ @@ -131,7 +133,7 @@ somefile.js: TypeError: baz is not a function undefined """ diff = None, None - self.assertEquals(diff, diff_fun(one, two)) + self.assertEqual(diff, diff_fun(one, two)) one = """ Still equal @@ -141,7 +143,7 @@ Extra line Still equal """ diff = '- Extra line', None - self.assertEquals(diff, diff_fun(one, two)) + self.assertEqual(diff, diff_fun(one, two)) one = """ Still equal @@ -151,7 +153,7 @@ Still equal Extra line """ diff = '+ Extra line', None - self.assertEquals(diff, diff_fun(one, two)) + self.assertEqual(diff, diff_fun(one, two)) one = """ undefined @@ -163,7 +165,7 @@ otherfile.js: TypeError: undefined is not a constructor """ diff = """- somefile.js: TypeError: undefined is not a constructor + otherfile.js: TypeError: undefined is not a constructor""", None - self.assertEquals(diff, diff_fun(one, two)) + self.assertEqual(diff, diff_fun(one, two)) # Test that skipping suppressions works. one = """ @@ -174,10 +176,10 @@ v8-foozzie source: foo v8-foozzie source: foo 42:TypeError: baz is not a function """ - self.assertEquals((None, 'foo'), diff_fun(one, two)) + self.assertEqual((None, 'foo'), diff_fun(one, two)) diff = """- 23:TypeError: bar is not a function + 42:TypeError: baz is not a function""", 'foo' - self.assertEquals(diff, diff_fun(one, two, skip=True)) + self.assertEqual(diff, diff_fun(one, two, skip=True)) def testOutputCapping(self): def output(stdout, is_crash): @@ -189,7 +191,7 @@ v8-foozzie source: foo capped_lines2): output1 = output(stdout1, is_crash1) output2 = output(stdout2, is_crash2) - self.assertEquals( + self.assertEqual( (capped_lines1, capped_lines2), v8_suppressions.get_output_capped(output1, output2)) @@ -229,6 +231,9 @@ def run_foozzie(second_d8_dir, *extra_flags, **kwargs): second_config = 'ignition_turbo' if 'second_config' in kwargs: second_config = 'jitless' + kwargs = {} + if PYTHON3: + kwargs['text'] = True return subprocess.check_output([ sys.executable, FOOZZIE, '--random-seed', '12345', @@ -237,8 +242,7 @@ def run_foozzie(second_d8_dir, *extra_flags, **kwargs): '--first-config', 'ignition', '--second-config', second_config, os.path.join(TEST_DATA, 'fuzz-123.js'), - ] + list(extra_flags)) - + ] + list(extra_flags), **kwargs) class SystemTest(unittest.TestCase): """This tests the whole correctness-fuzzing harness with fake build @@ -254,7 +258,7 @@ class SystemTest(unittest.TestCase): """ def testSyntaxErrorDiffPass(self): stdout = run_foozzie('build1', '--skip-sanity-checks') - self.assertEquals('# V8 correctness - pass\n', cut_verbose_output(stdout)) + self.assertEqual('# V8 correctness - pass\n', cut_verbose_output(stdout)) # Default comparison includes suppressions. self.assertIn('v8_suppressions.js', stdout) # Default comparison doesn't include any specific mock files. @@ -270,8 +274,8 @@ class SystemTest(unittest.TestCase): '--first-config-extra-flags=--flag2=0', '--second-config-extra-flags=--flag3') e = ctx.exception - self.assertEquals(v8_foozzie.RETURN_FAIL, e.returncode) - self.assertEquals(expected_output, cut_verbose_output(e.output)) + self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode) + self.assertEqual(expected_output, cut_verbose_output(e.output)) def testSanityCheck(self): with open(os.path.join(TEST_DATA, 'sanity_check_output.txt')) as f: @@ -279,8 +283,8 @@ class SystemTest(unittest.TestCase): with self.assertRaises(subprocess.CalledProcessError) as ctx: run_foozzie('build2') e = ctx.exception - self.assertEquals(v8_foozzie.RETURN_FAIL, e.returncode) - self.assertEquals(expected_output, e.output) + self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode) + self.assertEqual(expected_output, e.output) def testDifferentArch(self): """Test that the architecture-specific mocks are passed to both runs when @@ -319,7 +323,7 @@ class SystemTest(unittest.TestCase): run_foozzie( 'build1', '--skip-sanity-checks', '--skip-suppressions') e = ctx.exception - self.assertEquals(v8_foozzie.RETURN_FAIL, e.returncode) + self.assertEqual(v8_foozzie.RETURN_FAIL, e.returncode) self.assertNotIn('v8_suppressions.js', e.output) diff --git a/tools/clusterfuzz/v8_suppressions.py b/tools/clusterfuzz/v8_suppressions.py index 003c59848f..a3fa351f68 100644 --- a/tools/clusterfuzz/v8_suppressions.py +++ b/tools/clusterfuzz/v8_suppressions.py @@ -27,6 +27,13 @@ to silence a particular class of problems. import itertools import re +try: + # Python 3 + from itertools import zip_longest +except ImportError: + # Python 2 + from itertools import izip_longest as zip_longest + # Max line length for regular experessions checking for lines to ignore. MAX_LINE_LENGTH = 512 @@ -135,7 +142,7 @@ def get_output_capped(output1, output2): def line_pairs(lines): - return itertools.izip_longest( + return zip_longest( lines, itertools.islice(lines, 1, None), fillvalue=None) @@ -183,14 +190,14 @@ def diff_output(output1, output2, allowed, ignore1, ignore2): return all(not e.match(line) for e in ignore) return fun - lines1 = filter(useful_line(ignore1), output1) - lines2 = filter(useful_line(ignore2), output2) + lines1 = list(filter(useful_line(ignore1), output1)) + lines2 = list(filter(useful_line(ignore2), output2)) # This keeps track where we are in the original source file of the fuzz # test case. source = None - for ((line1, lookahead1), (line2, lookahead2)) in itertools.izip_longest( + for ((line1, lookahead1), (line2, lookahead2)) in zip_longest( line_pairs(lines1), line_pairs(lines2), fillvalue=(None, None)): # Only one of the two iterators should run out.