Fix python scripts to work with python3 (#2426)

Updated script to work with python3 and python2.

Added required tools.

We added a section to the readme to mention the tools that are needed to
build and test spirv-tools. For the compiler, the compilers used by the
bots are mentioned.

The bots have been changed. The windows bots will not use python 3.6 for testing. The other bots will still use python 2.7. Both Python2 and Python3 will be tested.

Fixes #2407.
Fixes #1856.

.appveyor.yml

@@ -42,7 +42,7 @@ install:
   - set NINJA_URL="https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-win.zip"
   - appveyor DownloadFile %NINJA_URL% -FileName ninja.zip
   - 7z x ninja.zip -oC:\ninja > nul
-  - set PATH=C:\ninja;%PATH%
+  - set PATH=C:\ninja;C:\Python36;%PATH%
 
 before_build:
   - git clone --depth=1 https://github.com/KhronosGroup/SPIRV-Headers.git external/spirv-headers

CMakeLists.txt

@@ -176,7 +176,7 @@ if(NOT COMMAND find_host_program)
   endmacro()
 endif()
 
-find_host_package(PythonInterp 2.7 REQUIRED)
+find_host_package(PythonInterp)
 
 # Check for symbol exports on Linux.
 # At the moment, this check will fail on the OSX build machines for the Android NDK.

README.md

@@ -269,6 +269,36 @@ cmake [-G <platform-generator>] <spirv-dir>
 Once the build files have been generated, build using your preferred
 development environment.
 
+### Tools you'll need
+
+For building and testing SPIRV-Tools, the following tools should be
+installed regardless of your OS:
+
+- [CMake](http://www.cmake.org/): for generating compilation targets.  Version
+  2.8.12 or later.
+- [Python](http://www.python.org/): for utility scripts and running the test 
+suite. Version 2 or 3.
+
+We will be moving to Python3 only in the future.  If you are using Python2, you
+will need to install Python-future: 
+```pip install future
+```
+
+SPIRV-Tools is regularly tested with the the following compilers:
+
+On Linux
+- GCC version 4.8.5
+- Clang version 3.8
+
+On MacOS
+- AppleClang 10.0
+
+On Windows
+- Visual Studio 2015
+- Visual Studio 2017
+
+Other compilers or later versions may work, but they are not tested.
+
 ### CMake options
 
 The following CMake options are supported:

kokoro/scripts/linux/build.sh

@@ -78,7 +78,7 @@ mkdir build && cd $SRC/build
 # Invoke the build.
 BUILD_SHA=${KOKORO_GITHUB_COMMIT:-$KOKORO_GITHUB_PULL_REQUEST_COMMIT}
 echo $(date): Starting build...
-cmake -GNinja -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_INSTALL_PREFIX=install -DRE2_BUILD_TESTING=OFF $ADDITIONAL_CMAKE_FLAGS $CMAKE_C_CXX_COMPILER ..
+cmake -DPYTHON_EXECUTABLE:FILEPATH=/usr/bin/python2.7 -GNinja -DCMAKE_BUILD_TYPE=$BUILD_TYPE -DCMAKE_INSTALL_PREFIX=install -DRE2_BUILD_TESTING=OFF $ADDITIONAL_CMAKE_FLAGS $CMAKE_C_CXX_COMPILER ..
 
 echo $(date): Build everything...
 ninja

kokoro/scripts/windows/build.bat

@@ -21,8 +21,8 @@ set SRC=%cd%\github\SPIRV-Tools
 set BUILD_TYPE=%1
 set VS_VERSION=%2
 
-:: Force usage of python 2.7 rather than 3.6
+:: Force usage of python 3.6
-set PATH=C:\python27;%PATH%
+set PATH=C:\python36;%PATH%
 
 cd %SRC%
 git clone --depth=1 https://github.com/KhronosGroup/SPIRV-Headers external/spirv-headers

test/tools/expect.py

@@ -18,15 +18,18 @@ as superclass and providing the expected_* variables required by the check_*()
 methods in the mixin classes.
 """
 import difflib
+import functools
 import os
 import re
 import subprocess
+import traceback
 from spirv_test_framework import SpirvTest
-
+from builtins import bytes
 
 def convert_to_unix_line_endings(source):
   """Converts all line endings in source to be unix line endings."""
-  return source.replace('\r\n', '\n').replace('\r', '\n')
+  result = source.replace('\r\n', '\n').replace('\r', '\n')
+  return result
 
 
 def substitute_file_extension(filename, extension):
@@ -133,7 +136,7 @@ class CorrectBinaryLengthAndPreamble(SpirvTest):
       word = binary[index * 4:(index + 1) * 4]
       if little_endian:
         word = reversed(word)
-      return reduce(lambda w, b: (w << 8) | ord(b), word, 0)
+      return functools.reduce(lambda w, b: (w << 8) | b, word, 0)
 
     def check_endianness(binary):
       """Checks the endianness of the given SPIR-V binary.
@@ -479,7 +482,7 @@ class ErrorMessageSubstr(SpirvTest):
     if not status.stderr:
       return False, 'Expected error message, but no output on stderr'
     if self.expected_error_substr not in convert_to_unix_line_endings(
-        status.stderr):
+        status.stderr.decode('utf8')):
       return False, ('Incorrect stderr output:\n{act}\n'
                      'Expected substring not found in stderr:\n{exp}'.format(
                          act=status.stderr, exp=self.expected_error_substr))
@@ -499,7 +502,7 @@ class WarningMessage(SpirvTest):
                      ' command execution')
     if not status.stderr:
       return False, 'Expected warning message, but no output on stderr'
-    if self.expected_warning != convert_to_unix_line_endings(status.stderr):
+    if self.expected_warning != convert_to_unix_line_endings(status.stderr.decode('utf8')):
       return False, ('Incorrect stderr output:\n{act}\n'
                      'Expected:\n{exp}'.format(
                          act=status.stderr, exp=self.expected_warning))
@@ -559,16 +562,16 @@ class StdoutMatch(SpirvTest):
       if not status.stdout:
         return False, 'Expected something on stdout'
     elif type(self.expected_stdout) == str:
-      if self.expected_stdout != convert_to_unix_line_endings(status.stdout):
+      if self.expected_stdout != convert_to_unix_line_endings(status.stdout.decode('utf8')):
         return False, ('Incorrect stdout output:\n{ac}\n'
                        'Expected:\n{ex}'.format(
                            ac=status.stdout, ex=self.expected_stdout))
     else:
-      if not self.expected_stdout.search(
+      converted = convert_to_unix_line_endings(status.stdout.decode('utf8'))
-          convert_to_unix_line_endings(status.stdout)):
+      if not self.expected_stdout.search(converted):
         return False, ('Incorrect stdout output:\n{ac}\n'
                        'Expected to match regex:\n{ex}'.format(
-                           ac=status.stdout, ex=self.expected_stdout.pattern))
+                           ac=status.stdout.decode('utf8'), ex=self.expected_stdout.pattern))
     return True, ''
 
 
@@ -593,13 +596,13 @@ class StderrMatch(SpirvTest):
       if not status.stderr:
         return False, 'Expected something on stderr'
     elif type(self.expected_stderr) == str:
-      if self.expected_stderr != convert_to_unix_line_endings(status.stderr):
+      if self.expected_stderr != convert_to_unix_line_endings(status.stderr.decode('utf8')):
         return False, ('Incorrect stderr output:\n{ac}\n'
                        'Expected:\n{ex}'.format(
                            ac=status.stderr, ex=self.expected_stderr))
     else:
       if not self.expected_stderr.search(
-          convert_to_unix_line_endings(status.stderr)):
+          convert_to_unix_line_endings(status.stderr.decode('utf8'))):
         return False, ('Incorrect stderr output:\n{ac}\n'
                        'Expected to match regex:\n{ex}'.format(
                            ac=status.stderr, ex=self.expected_stderr.pattern))
@@ -664,7 +667,7 @@ class ExecutedListOfPasses(SpirvTest):
     # Collect all the output lines containing a pass name.
     pass_names = []
     pass_name_re = re.compile(r'.*IR before pass (?P<pass_name>[\S]+)')
-    for line in status.stderr.splitlines():
+    for line in status.stderr.decode('utf8').splitlines():
       match = pass_name_re.match(line)
       if match:
         pass_names.append(match.group('pass_name'))