AuroraMiddleware/SPIRV-Tools
1
0
Fork 0
mirror of https://github.com/KhronosGroup/SPIRV-Tools synced 2024-10-18 11:10:05 +00:00
Code Issues Packages Projects Releases Wiki Activity

Add testing framework for tools.

Browse Source 
This forks the testing harness from https://github.com/google/shaderc
to allow testing CLI tools.

New features needed for SPIRV-Tools include:

1- A new PlaceHolder subclass for spirv shaders.  This place holder
   calls spirv-as to convert assembly input into SPIRV bytecode. This is
   required for most tools in SPIRV-Tools.

2- A minimal testing file for testing basic functionality of spirv-opt.

Add tests for all flags in spirv-opt.

1. Adds tests to check that known flags match the names that each pass
   advertises.
2. Adds tests to check that -O, -Os and --legalize-hlsl schedule the
   expected passes.
3. Adds more functionality to Expect classes to support regular
   expression matching on stderr.
4. Add checks for integer arguments to optimization flags.
5. Fixes #1817 by modifying the parsing of integer arguments in
   flags that take them.
6. Fixes -Oconfig file parsing (#1778). It reads every line of the file
   into a string and then parses that string by tokenizing every group of
   characters between whitespaces (using the standard cin reading
   operator).  This mimics shell command-line parsing, but it does not
   support quoting (and I'm not planning to).
This commit is contained in:
Diego Novillo 2018-08-03 18:08:46 -04:00
parent 1bdade77ea
commit 03000a3a38
19 changed files with 1987 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
CMakeLists.txt
View File

@ -29,6 +29,7 @@ enable_testing()
set(SPIRV_TOOLS "SPIRV-Tools")
include(GNUInstallDirs)
include(cmake/setup_build.cmake)
set(CMAKE_POSITION_INDEPENDENT_CODE ON)

20
cmake/setup_build.cmake Normal file
View File

@ -0,0 +1,20 @@
# Find nosetests; see spirv_add_nosetests() for opting in to nosetests in a
# specific directory.
find_program(NOSETESTS_EXE NAMES nosetests PATHS $ENV{PYTHON_PACKAGE_PATH})
if (NOT NOSETESTS_EXE)
message(STATUS "SPIRV-Tools: nosetests was not found - python support code will not be tested")
else()
message(STATUS "SPIRV-Tools: nosetests found - python support code will be tested")
endif()
# Run nosetests on file ${PREFIX}_nosetest.py. Nosetests will look for classes
# and functions whose names start with "nosetest". The test name will be
# ${PREFIX}_nosetests.
function(spirv_add_nosetests PREFIX)
if(NOT "${SPIRV_SKIP_TESTS}" AND NOSETESTS_EXE)
add_test(
NAME ${PREFIX}_nosetests
COMMAND ${NOSETESTS_EXE} -m "^[Nn]ose[Tt]est" -v
${CMAKE_CURRENT_SOURCE_DIR}/${PREFIX}_nosetest.py)
endif()
endfunction()

2
source/opt/dead_variable_elimination.h
View File

@ -26,7 +26,7 @@ namespace opt {
class DeadVariableElimination : public MemPass {
public:
const char* name() const override { return "dead-variable-elimination"; }
const char* name() const override { return "eliminate-dead-variables"; }
Status Process() override;
IRContext::Analysis GetPreservedAnalyses() override {

2
source/opt/flatten_decoration_pass.h
View File

@ -25,7 +25,7 @@ namespace opt {
// See optimizer.hpp for documentation.
class FlattenDecorationPass : public Pass {
public:
const char* name() const override { return "flatten-decoration"; }
const char* name() const override { return "flatten-decorations"; }
Status Process() override;
};

2
source/opt/loop_fission.h
View File

@ -55,7 +55,7 @@ class LoopFissionPass : public Pass {
bool split_multiple_times = true)
: split_criteria_(functor), split_multiple_times_(split_multiple_times) {}
const char* name() const override { return "Loop Fission"; }
const char* name() const override { return "loop-fission"; }
Pass::Status Process() override;

2
source/opt/loop_unroller.h
View File

@ -26,7 +26,7 @@ class LoopUnroller : public Pass {
LoopUnroller(bool fully_unroll, int unroll_factor)
: Pass(), fully_unroll_(fully_unroll), unroll_factor_(unroll_factor) {}
const char* name() const override { return "Loop unroller"; }
const char* name() const override { return "loop-unroll"; }
Status Process() override;

19
source/opt/optimizer.cpp
View File

@ -119,7 +119,7 @@ Optimizer& Optimizer::RegisterLegalizationPasses() {
.RegisterPass(CreateLocalSingleBlockLoadStoreElimPass())
.RegisterPass(CreateLocalSingleStoreElimPass())
.RegisterPass(CreateAggressiveDCEPass())
// Split up aggragates so they are easier to deal with.
// Split up aggregates so they are easier to deal with.
.RegisterPass(CreateScalarReplacementPass(0))
// Remove loads and stores so everything is in intermediate values.
// Takes care of copy propagation of non-members.
@ -348,7 +348,7 @@ bool Optimizer::RegisterPassFromFlag(const std::string& flag) {
if (pass_args.size() == 0) {
RegisterPass(CreateScalarReplacementPass());
} else {
uint32_t limit = atoi(pass_args.c_str());
int limit = atoi(pass_args.c_str());
if (limit > 0) {
RegisterPass(CreateScalarReplacementPass(limit));
} else {
@ -409,7 +409,7 @@ bool Optimizer::RegisterPassFromFlag(const std::string& flag) {
CreateLoopFusionPass(static_cast<size_t>(max_registers_per_loop)));
} else {
Error(consumer(), nullptr, {},
"--loop-fusion must be have a positive integer argument");
"--loop-fusion must have a positive integer argument");
return false;
}
} else if (pass_name == "loop-unroll") {
@ -418,15 +418,24 @@ bool Optimizer::RegisterPassFromFlag(const std::string& flag) {
RegisterPass(CreateVectorDCEPass());
} else if (pass_name == "loop-unroll-partial") {
int factor = (pass_args.size() > 0) ? atoi(pass_args.c_str()) : 0;
if (factor != 0) {
if (factor > 0) {
RegisterPass(CreateLoopUnrollPass(false, factor));
} else {
Error(consumer(), nullptr, {},
"--loop-unroll-partial must have a non-0 integer argument");
"--loop-unroll-partial must have a positive integer argument");
return false;
}
} else if (pass_name == "loop-peeling") {
RegisterPass(CreateLoopPeelingPass());
} else if (pass_name == "loop-peeling-threshold") {
int factor = (pass_args.size() > 0) ? atoi(pass_args.c_str()) : 0;
if (factor > 0) {
opt::LoopPeelingPass::SetLoopPeelingThreshold(factor);
} else {
Error(consumer(), nullptr, {},
"--loop-peeling-threshold must have a positive integer argument");
return false;
}
} else if (pass_name == "ccp") {
RegisterPass(CreateCCPPass());
} else if (pass_name == "O") {

2
source/opt/replace_invalid_opc.h
View File

@ -28,7 +28,7 @@ namespace opt {
// value, the instruction will simply be deleted.
class ReplaceInvalidOpcodePass : public Pass {
public:
const char* name() const override { return "replace-invalid-opcodes"; }
const char* name() const override { return "replace-invalid-opcode"; }
Status Process() override;
private:

1
test/CMakeLists.txt
View File

@ -221,5 +221,6 @@ add_subdirectory(comp)
add_subdirectory(link)
add_subdirectory(opt)
add_subdirectory(stats)
add_subdirectory(tools)
add_subdirectory(util)
add_subdirectory(val)

18
test/tools/CMakeLists.txt Normal file
View File

@ -0,0 +1,18 @@
# Copyright (c) 2018 Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
spirv_add_nosetests(expect)
spirv_add_nosetests(spirv_test_framework)
add_subdirectory(opt)

677
test/tools/expect.py Executable file
View File

@ -0,0 +1,677 @@
# Copyright (c) 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A number of common spirv result checks coded in mixin classes.
A test case can use these checks by declaring their enclosing mixin classes
as superclass and providing the expected_* variables required by the check_*()
methods in the mixin classes.
"""
import difflib
import os
import re
import subprocess
from spirv_test_framework import SpirvTest
def convert_to_unix_line_endings(source):
"""Converts all line endings in source to be unix line endings."""
return source.replace('\r\n', '\n').replace('\r', '\n')
def substitute_file_extension(filename, extension):
"""Substitutes file extension, respecting known shader extensions.
foo.vert -> foo.vert.[extension] [similarly for .frag, .comp, etc.]
foo.glsl -> foo.[extension]
foo.unknown -> foo.[extension]
foo -> foo.[extension]
"""
if filename[-5:] not in [
'.vert', '.frag', '.tesc', '.tese', '.geom', '.comp', '.spvasm'
]:
return filename.rsplit('.', 1)[0] + '.' + extension
else:
return filename + '.' + extension
def get_object_filename(source_filename):
"""Gets the object filename for the given source file."""
return substitute_file_extension(source_filename, 'spv')
def get_assembly_filename(source_filename):
"""Gets the assembly filename for the given source file."""
return substitute_file_extension(source_filename, 'spvasm')
def verify_file_non_empty(filename):
"""Checks that a given file exists and is not empty."""
if not os.path.isfile(filename):
return False, 'Cannot find file: ' + filename
if not os.path.getsize(filename):
return False, 'Empty file: ' + filename
return True, ''
class ReturnCodeIsZero(SpirvTest):
"""Mixin class for checking that the return code is zero."""
def check_return_code_is_zero(self, status):
if status.returncode:
return False, 'Non-zero return code: {ret}\n'.format(
ret=status.returncode)
return True, ''
class NoOutputOnStdout(SpirvTest):
"""Mixin class for checking that there is no output on stdout."""
def check_no_output_on_stdout(self, status):
if status.stdout:
return False, 'Non empty stdout: {out}\n'.format(out=status.stdout)
return True, ''
class NoOutputOnStderr(SpirvTest):
"""Mixin class for checking that there is no output on stderr."""
def check_no_output_on_stderr(self, status):
if status.stderr:
return False, 'Non empty stderr: {err}\n'.format(err=status.stderr)
return True, ''
class SuccessfulReturn(ReturnCodeIsZero, NoOutputOnStdout, NoOutputOnStderr):
"""Mixin class for checking that return code is zero and no output on
stdout and stderr."""
pass
class NoGeneratedFiles(SpirvTest):
"""Mixin class for checking that there is no file generated."""
def check_no_generated_files(self, status):
all_files = os.listdir(status.directory)
input_files = status.input_filenames
if all([f.startswith(status.directory) for f in input_files]):
all_files = [os.path.join(status.directory, f) for f in all_files]
generated_files = set(all_files) - set(input_files)
if len(generated_files) == 0:
return True, ''
else:
return False, 'Extra files generated: {}'.format(generated_files)
class CorrectBinaryLengthAndPreamble(SpirvTest):
"""Provides methods for verifying preamble for a SPIR-V binary."""
def verify_binary_length_and_header(self, binary, spv_version=0x10000):
"""Checks that the given SPIR-V binary has valid length and header.
Returns:
False, error string if anything is invalid
True, '' otherwise
Args:
binary: a bytes object containing the SPIR-V binary
spv_version: target SPIR-V version number, with same encoding
as the version word in a SPIR-V header.
"""
def read_word(binary, index, little_endian):
"""Reads the index-th word from the given binary file."""
word = binary[index * 4:(index + 1) * 4]
if little_endian:
word = reversed(word)
return reduce(lambda w, b: (w << 8) | ord(b), word, 0)
def check_endianness(binary):
"""Checks the endianness of the given SPIR-V binary.
Returns:
True if it's little endian, False if it's big endian.
None if magic number is wrong.
"""
first_word = read_word(binary, 0, True)
if first_word == 0x07230203:
return True
first_word = read_word(binary, 0, False)
if first_word == 0x07230203:
return False
return None
num_bytes = len(binary)
if num_bytes % 4 != 0:
return False, ('Incorrect SPV binary: size should be a multiple'
' of words')
if num_bytes < 20:
return False, 'Incorrect SPV binary: size less than 5 words'
preamble = binary[0:19]
little_endian = check_endianness(preamble)
# SPIR-V module magic number
if little_endian is None:
return False, 'Incorrect SPV binary: wrong magic number'
# SPIR-V version number
version = read_word(preamble, 1, little_endian)
# TODO(dneto): Recent Glslang uses version word 0 for opengl_compat
# profile
if version != spv_version and version != 0:
return False, 'Incorrect SPV binary: wrong version number'
# Shaderc-over-Glslang (0x000d....) or
# SPIRV-Tools (0x0007....) generator number
if read_word(preamble, 2, little_endian) != 0x000d0007 and \
read_word(preamble, 2, little_endian) != 0x00070000:
return False, ('Incorrect SPV binary: wrong generator magic ' 'number')
# reserved for instruction schema
if read_word(preamble, 4, little_endian) != 0:
return False, 'Incorrect SPV binary: the 5th byte should be 0'
return True, ''
class CorrectObjectFilePreamble(CorrectBinaryLengthAndPreamble):
"""Provides methods for verifying preamble for a SPV object file."""
def verify_object_file_preamble(self, filename, spv_version=0x10000):
"""Checks that the given SPIR-V binary file has correct preamble."""
success, message = verify_file_non_empty(filename)
if not success:
return False, message
with open(filename, 'rb') as object_file:
object_file.seek(0, os.SEEK_END)
num_bytes = object_file.tell()
object_file.seek(0)
binary = bytes(object_file.read())
return self.verify_binary_length_and_header(binary, spv_version)
return True, ''
class CorrectAssemblyFilePreamble(SpirvTest):
"""Provides methods for verifying preamble for a SPV assembly file."""
def verify_assembly_file_preamble(self, filename):
success, message = verify_file_non_empty(filename)
if not success:
return False, message
with open(filename) as assembly_file:
line1 = assembly_file.readline()
line2 = assembly_file.readline()
line3 = assembly_file.readline()
if (line1 != '; SPIR-V\n' or line2 != '; Version: 1.0\n' or
(not line3.startswith('; Generator: Google Shaderc over Glslang;'))):
return False, 'Incorrect SPV assembly'
return True, ''
class ValidObjectFile(SuccessfulReturn, CorrectObjectFilePreamble):
"""Mixin class for checking that every input file generates a valid SPIR-V 1.0
object file following the object file naming rule, and there is no output on
stdout/stderr."""
def check_object_file_preamble(self, status):
for input_filename in status.input_filenames:
object_filename = get_object_filename(input_filename)
success, message = self.verify_object_file_preamble(
os.path.join(status.directory, object_filename))
if not success:
return False, message
return True, ''
class ValidObjectFile1_3(ReturnCodeIsZero, CorrectObjectFilePreamble):
"""Mixin class for checking that every input file generates a valid SPIR-V 1.3
object file following the object file naming rule, and there is no output on
stdout/stderr."""
def check_object_file_preamble(self, status):
for input_filename in status.input_filenames:
object_filename = get_object_filename(input_filename)
success, message = self.verify_object_file_preamble(
os.path.join(status.directory, object_filename), 0x10300)
if not success:
return False, message
return True, ''
class ValidObjectFileWithAssemblySubstr(SuccessfulReturn,
CorrectObjectFilePreamble):
"""Mixin class for checking that every input file generates a valid object
file following the object file naming rule, there is no output on
stdout/stderr, and the disassmbly contains a specified substring per
input.
"""
def check_object_file_disassembly(self, status):
for an_input in status.inputs:
object_filename = get_object_filename(an_input.filename)
obj_file = str(os.path.join(status.directory, object_filename))
success, message = self.verify_object_file_preamble(obj_file)
if not success:
return False, message
cmd = [status.test_manager.disassembler_path, '--no-color', obj_file]
process = subprocess.Popen(
args=cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=status.directory)
output = process.communicate(None)
disassembly = output[0]
if not isinstance(an_input.assembly_substr, str):
return False, 'Missing assembly_substr member'
if an_input.assembly_substr not in disassembly:
return False, ('Incorrect disassembly output:\n{asm}\n'
'Expected substring not found:\n{exp}'.format(
asm=disassembly, exp=an_input.assembly_substr))
return True, ''
class ValidNamedObjectFile(SuccessfulReturn, CorrectObjectFilePreamble):
"""Mixin class for checking that a list of object files with the given
names are correctly generated, and there is no output on stdout/stderr.
To mix in this class, subclasses need to provide expected_object_filenames
as the expected object filenames.
"""
def check_object_file_preamble(self, status):
for object_filename in self.expected_object_filenames:
success, message = self.verify_object_file_preamble(
os.path.join(status.directory, object_filename))
if not success:
return False, message
return True, ''
class ValidFileContents(SpirvTest):
"""Mixin class to test that a specific file contains specific text
To mix in this class, subclasses need to provide expected_file_contents as
the contents of the file and target_filename to determine the location."""
def check_file(self, status):
target_filename = os.path.join(status.directory, self.target_filename)
if not os.path.isfile(target_filename):
return False, 'Cannot find file: ' + target_filename
with open(target_filename, 'r') as target_file:
file_contents = target_file.read()
if isinstance(self.expected_file_contents, str):
if file_contents == self.expected_file_contents:
return True, ''
return False, ('Incorrect file output: \n{act}\n'
'Expected:\n{exp}'
'With diff:\n{diff}'.format(
act=file_contents,
exp=self.expected_file_contents,
diff='\n'.join(
list(
difflib.unified_diff(
self.expected_file_contents.split('\n'),
file_contents.split('\n'),
fromfile='expected_output',
tofile='actual_output')))))
elif isinstance(self.expected_file_contents, type(re.compile(''))):
if self.expected_file_contents.search(file_contents):
return True, ''
return False, ('Incorrect file output: \n{act}\n'
'Expected matching regex pattern:\n{exp}'.format(
act=file_contents,
exp=self.expected_file_contents.pattern))
return False, (
'Could not open target file ' + target_filename + ' for reading')
class ValidAssemblyFile(SuccessfulReturn, CorrectAssemblyFilePreamble):
"""Mixin class for checking that every input file generates a valid assembly
file following the assembly file naming rule, and there is no output on
stdout/stderr."""
def check_assembly_file_preamble(self, status):
for input_filename in status.input_filenames:
assembly_filename = get_assembly_filename(input_filename)
success, message = self.verify_assembly_file_preamble(
os.path.join(status.directory, assembly_filename))
if not success:
return False, message
return True, ''
class ValidAssemblyFileWithSubstr(ValidAssemblyFile):
"""Mixin class for checking that every input file generates a valid assembly
file following the assembly file naming rule, there is no output on
stdout/stderr, and all assembly files have the given substring specified
by expected_assembly_substr.
To mix in this class, subclasses need to provde expected_assembly_substr
as the expected substring.
"""
def check_assembly_with_substr(self, status):
for input_filename in status.input_filenames:
assembly_filename = get_assembly_filename(input_filename)
success, message = self.verify_assembly_file_preamble(
os.path.join(status.directory, assembly_filename))
if not success:
return False, message
with open(assembly_filename, 'r') as f:
content = f.read()
if self.expected_assembly_substr not in convert_to_unix_line_endings(
content):
return False, ('Incorrect assembly output:\n{asm}\n'
'Expected substring not found:\n{exp}'.format(
asm=content, exp=self.expected_assembly_substr))
return True, ''
class ValidAssemblyFileWithoutSubstr(ValidAssemblyFile):
"""Mixin class for checking that every input file generates a valid assembly
file following the assembly file naming rule, there is no output on
stdout/stderr, and no assembly files have the given substring specified
by unexpected_assembly_substr.
To mix in this class, subclasses need to provde unexpected_assembly_substr
as the substring we expect not to see.
"""
def check_assembly_for_substr(self, status):
for input_filename in status.input_filenames:
assembly_filename = get_assembly_filename(input_filename)
success, message = self.verify_assembly_file_preamble(
os.path.join(status.directory, assembly_filename))
if not success:
return False, message
with open(assembly_filename, 'r') as f:
content = f.read()
if self.unexpected_assembly_substr in convert_to_unix_line_endings(
content):
return False, ('Incorrect assembly output:\n{asm}\n'
'Unexpected substring found:\n{unexp}'.format(
asm=content, exp=self.unexpected_assembly_substr))
return True, ''
class ValidNamedAssemblyFile(SuccessfulReturn, CorrectAssemblyFilePreamble):
"""Mixin class for checking that a list of assembly files with the given
names are correctly generated, and there is no output on stdout/stderr.
To mix in this class, subclasses need to provide expected_assembly_filenames
as the expected assembly filenames.
"""
def check_object_file_preamble(self, status):
for assembly_filename in self.expected_assembly_filenames:
success, message = self.verify_assembly_file_preamble(
os.path.join(status.directory, assembly_filename))
if not success:
return False, message
return True, ''
class ErrorMessage(SpirvTest):
"""Mixin class for tests that fail with a specific error message.
To mix in this class, subclasses need to provide expected_error as the
expected error message.
The test should fail if the subprocess was terminated by a signal.
"""
def check_has_error_message(self, status):
if not status.returncode:
return False, ('Expected error message, but returned success from '
'command execution')
if status.returncode < 0:
# On Unix, a negative value -N for Popen.returncode indicates
# termination by signal N.
# https://docs.python.org/2/library/subprocess.html
return False, ('Expected error message, but command was terminated by '
'signal ' + str(status.returncode))
if not status.stderr:
return False, 'Expected error message, but no output on stderr'
if self.expected_error != convert_to_unix_line_endings(status.stderr):
return False, ('Incorrect stderr output:\n{act}\n'
'Expected:\n{exp}'.format(
act=status.stderr, exp=self.expected_error))
return True, ''
class ErrorMessageSubstr(SpirvTest):
"""Mixin class for tests that fail with a specific substring in the error
message.
To mix in this class, subclasses need to provide expected_error_substr as
the expected error message substring.
The test should fail if the subprocess was terminated by a signal.
"""
def check_has_error_message_as_substring(self, status):
if not status.returncode:
return False, ('Expected error message, but returned success from '
'command execution')
if status.returncode < 0:
# On Unix, a negative value -N for Popen.returncode indicates
# termination by signal N.
# https://docs.python.org/2/library/subprocess.html
return False, ('Expected error message, but command was terminated by '
'signal ' + str(status.returncode))
if not status.stderr:
return False, 'Expected error message, but no output on stderr'
if self.expected_error_substr not in convert_to_unix_line_endings(
status.stderr):
return False, ('Incorrect stderr output:\n{act}\n'
'Expected substring not found in stderr:\n{exp}'.format(
act=status.stderr, exp=self.expected_error_substr))
return True, ''
class WarningMessage(SpirvTest):
"""Mixin class for tests that succeed but have a specific warning message.
To mix in this class, subclasses need to provide expected_warning as the
expected warning message.
"""
def check_has_warning_message(self, status):
if status.returncode:
return False, ('Expected warning message, but returned failure from'
' command execution')
if not status.stderr:
return False, 'Expected warning message, but no output on stderr'
if self.expected_warning != convert_to_unix_line_endings(status.stderr):
return False, ('Incorrect stderr output:\n{act}\n'
'Expected:\n{exp}'.format(
act=status.stderr, exp=self.expected_warning))
return True, ''
class ValidObjectFileWithWarning(NoOutputOnStdout, CorrectObjectFilePreamble,
WarningMessage):
"""Mixin class for checking that every input file generates a valid object
file following the object file naming rule, with a specific warning message.
"""
def check_object_file_preamble(self, status):
for input_filename in status.input_filenames:
object_filename = get_object_filename(input_filename)
success, message = self.verify_object_file_preamble(
os.path.join(status.directory, object_filename))
if not success:
return False, message
return True, ''
class ValidAssemblyFileWithWarning(NoOutputOnStdout,
CorrectAssemblyFilePreamble, WarningMessage):
"""Mixin class for checking that every input file generates a valid assembly
file following the assembly file naming rule, with a specific warning
message."""
def check_assembly_file_preamble(self, status):
for input_filename in status.input_filenames:
assembly_filename = get_assembly_filename(input_filename)
success, message = self.verify_assembly_file_preamble(
os.path.join(status.directory, assembly_filename))
if not success:
return False, message
return True, ''
class StdoutMatch(SpirvTest):
"""Mixin class for tests that can expect output on stdout.
To mix in this class, subclasses need to provide expected_stdout as the
expected stdout output.
For expected_stdout, if it's True, then they expect something on stdout but
will not check what it is. If it's a string, expect an exact match. If it's
anything else, it is assumed to be a compiled regular expression which will
be matched against re.search(). It will expect
expected_stdout.search(status.stdout) to be true.
"""
def check_stdout_match(self, status):
# "True" in this case means we expect something on stdout, but we do not
# care what it is, we want to distinguish this from "blah" which means we
# expect exactly the string "blah".
if self.expected_stdout is True:
if not status.stdout:
return False, 'Expected something on stdout'
elif type(self.expected_stdout) == str:
if self.expected_stdout != convert_to_unix_line_endings(status.stdout):
return False, ('Incorrect stdout output:\n{ac}\n'
'Expected:\n{ex}'.format(
ac=status.stdout, ex=self.expected_stdout))
else:
if not self.expected_stdout.search(
convert_to_unix_line_endings(status.stdout)):
return False, ('Incorrect stdout output:\n{ac}\n'
'Expected to match regex:\n{ex}'.format(
ac=status.stdout, ex=self.expected_stdout.pattern))
return True, ''
class StderrMatch(SpirvTest):
"""Mixin class for tests that can expect output on stderr.
To mix in this class, subclasses need to provide expected_stderr as the
expected stderr output.
For expected_stderr, if it's True, then they expect something on stderr,
but will not check what it is. If it's a string, expect an exact match.
If it's anything else, it is assumed to be a compiled regular expression
which will be matched against re.search(). It will expect
expected_stderr.search(status.stderr) to be true.
"""
def check_stderr_match(self, status):
# "True" in this case means we expect something on stderr, but we do not
# care what it is, we want to distinguish this from "blah" which means we
# expect exactly the string "blah".
if self.expected_stderr is True:
if not status.stderr:
return False, 'Expected something on stderr'
elif type(self.expected_stderr) == str:
if self.expected_stderr != convert_to_unix_line_endings(status.stderr):
return False, ('Incorrect stderr output:\n{ac}\n'
'Expected:\n{ex}'.format(
ac=status.stderr, ex=self.expected_stderr))
else:
if not self.expected_stderr.search(
convert_to_unix_line_endings(status.stderr)):
return False, ('Incorrect stderr output:\n{ac}\n'
'Expected to match regex:\n{ex}'.format(
ac=status.stderr, ex=self.expected_stderr.pattern))
return True, ''
class StdoutNoWiderThan80Columns(SpirvTest):
"""Mixin class for tests that require stdout to 80 characters or narrower.
To mix in this class, subclasses need to provide expected_stdout as the
expected stdout output.
"""
def check_stdout_not_too_wide(self, status):
if not status.stdout:
return True, ''
else:
for line in status.stdout.splitlines():
if len(line) > 80:
return False, ('Stdout line longer than 80 columns: %s' % line)
return True, ''
class NoObjectFile(SpirvTest):
"""Mixin class for checking that no input file has a corresponding object
file."""
def check_no_object_file(self, status):
for input_filename in status.input_filenames:
object_filename = get_object_filename(input_filename)
full_object_file = os.path.join(status.directory, object_filename)
print('checking %s' % full_object_file)
if os.path.isfile(full_object_file):
return False, (
'Expected no object file, but found: %s' % full_object_file)
return True, ''
class NoNamedOutputFiles(SpirvTest):
"""Mixin class for checking that no specified output files exist.
The expected_output_filenames member should be full pathnames."""
def check_no_named_output_files(self, status):
for object_filename in self.expected_output_filenames:
if os.path.isfile(object_filename):
return False, (
'Expected no output file, but found: %s' % object_filename)
return True, ''
class ExecutedListOfPasses(SpirvTest):
"""Mixin class for checking that a list of passes where executed.
It works by analyzing the output of the --print-all flag to spirv-opt.
For this mixin to work, the class member expected_passes should be a sequence
of pass names as returned by Pass::name().
"""
def check_list_of_executed_passes(self, status):
# Collect all the output lines containing a pass name.
pass_names = []
pass_name_re = re.compile(r'.*IR before pass (?P<pass_name>[\S]+)')
for line in status.stderr.splitlines():
match = pass_name_re.match(line)
if match:
pass_names.append(match.group('pass_name'))
for (expected, actual) in zip(self.expected_passes, pass_names):
if expected != actual:
return False, (
'Expected pass "%s" but found pass "%s"\n' % (expected, actual))
return True, ''

80
test/tools/expect_nosetest.py Executable file
View File

@ -0,0 +1,80 @@
# Copyright (c) 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for the expect module."""
import expect
from spirv_test_framework import TestStatus
from nose.tools import assert_equal, assert_true, assert_false
import re
def nosetest_get_object_name():
"""Tests get_object_filename()."""
source_and_object_names = [('a.vert', 'a.vert.spv'), ('b.frag', 'b.frag.spv'),
('c.tesc', 'c.tesc.spv'), ('d.tese', 'd.tese.spv'),
('e.geom', 'e.geom.spv'), ('f.comp', 'f.comp.spv'),
('file', 'file.spv'), ('file.', 'file.spv'),
('file.uk',
'file.spv'), ('file.vert.',
'file.vert.spv'), ('file.vert.bla',
'file.vert.spv')]
actual_object_names = [
expect.get_object_filename(f[0]) for f in source_and_object_names
]
expected_object_names = [f[1] for f in source_and_object_names]
assert_equal(actual_object_names, expected_object_names)
class TestStdoutMatchADotC(expect.StdoutMatch):
expected_stdout = re.compile('a.c')
def nosetest_stdout_match_regex_has_match():
test = TestStdoutMatchADotC()
status = TestStatus(
test_manager=None,
returncode=0,
stdout='0abc1',
stderr=None,
directory=None,
inputs=None,
input_filenames=None)
assert_true(test.check_stdout_match(status)[0])
def nosetest_stdout_match_regex_no_match():
test = TestStdoutMatchADotC()
status = TestStatus(
test_manager=None,
returncode=0,
stdout='ab',
stderr=None,
directory=None,
inputs=None,
input_filenames=None)
assert_false(test.check_stdout_match(status)[0])
def nosetest_stdout_match_regex_empty_stdout():
test = TestStdoutMatchADotC()
status = TestStatus(
test_manager=None,
returncode=0,
stdout='',
stderr=None,
directory=None,
inputs=None,
input_filenames=None)
assert_false(test.check_stdout_match(status)[0])

25
test/tools/opt/CMakeLists.txt Normal file
View File

@ -0,0 +1,25 @@
# Copyright (c) 2018 Google LLC.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
if(NOT ${SPIRV_SKIP_TESTS})
if(${PYTHONINTERP_FOUND})
add_test(NAME spirv_opt_tests
COMMAND ${PYTHON_EXECUTABLE}
${CMAKE_CURRENT_SOURCE_DIR}/../spirv_test_framework.py
$<TARGET_FILE:spirv-opt> $<TARGET_FILE:spirv-as> $<TARGET_FILE:spirv-dis>
--test-dir ${CMAKE_CURRENT_SOURCE_DIR})
else()
message("Skipping CLI tools tests - Python executable not found")
endif()
endif()

330
test/tools/opt/flags.py Normal file
View File

@ -0,0 +1,330 @@
# Copyright (c) 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import placeholder
import expect
import re
from spirv_test_framework import inside_spirv_testsuite
def empty_main_assembly():
return """
OpCapability Shader
OpMemoryModel Logical GLSL450
OpEntryPoint Vertex %4 "main"
OpName %4 "main"
%2 = OpTypeVoid
%3 = OpTypeFunction %2
%4 = OpFunction %2 None %3
%5 = OpLabel
OpReturn
OpFunctionEnd"""
@inside_spirv_testsuite('SpirvOptBase')
class TestAssemblyFileAsOnlyParameter(expect.ValidObjectFile1_3):
"""Tests that spirv-opt accepts a SPIR-V object file."""
shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm')
output = placeholder.TempFileName('output.spv')
spirv_args = [shader, '-o', output]
expected_object_filenames = (output)
@inside_spirv_testsuite('SpirvOptFlags')
class TestHelpFlag(expect.ReturnCodeIsZero, expect.StdoutMatch):
"""Test the --help flag."""
spirv_args = ['--help']
expected_stdout = re.compile(r'.*The SPIR-V binary is read from <input>')
@inside_spirv_testsuite('SpirvOptFlags')
class TestValidPassFlags(expect.ValidObjectFile1_3,
expect.ExecutedListOfPasses):
"""Tests that spirv-opt accepts all valid optimization flags."""
flags = [
'--ccp', '--cfg-cleanup', '--combine-access-chains', '--compact-ids',
'--convert-local-access-chains', '--copy-propagate-arrays',
'--eliminate-common-uniform', '--eliminate-dead-branches',
'--eliminate-dead-code-aggressive', '--eliminate-dead-const',
'--eliminate-dead-functions', '--eliminate-dead-inserts',
'--eliminate-dead-variables', '--eliminate-insert-extract',
'--eliminate-local-multi-store', '--eliminate-local-single-block',
'--eliminate-local-single-store', '--flatten-decorations',
'--fold-spec-const-op-composite', '--freeze-spec-const',
'--if-conversion', '--inline-entry-points-exhaustive', '--loop-fission',
'20', '--loop-fusion', '5', '--loop-unroll', '--loop-unroll-partial', '3',
'--loop-peeling', '--merge-blocks', '--merge-return', '--loop-unswitch',
'--private-to-local', '--reduce-load-size', '--redundancy-elimination',
'--remove-duplicates', '--replace-invalid-opcode', '--ssa-rewrite',
'--scalar-replacement', '--scalar-replacement=42', '--strength-reduction',
'--strip-debug', '--strip-reflect', '--vector-dce', '--workaround-1209',
'--unify-const'
]
expected_passes = [
'ccp',
'cfg-cleanup',
'combine-access-chains',
'compact-ids',
'convert-local-access-chains',
'copy-propagate-arrays',
'eliminate-common-uniform',
'eliminate-dead-branches',
'eliminate-dead-code-aggressive',
'eliminate-dead-const',
'eliminate-dead-functions',
'eliminate-dead-inserts',
'eliminate-dead-variables',
# --eliminate-insert-extract runs the simplify-instructions pass.
'simplify-instructions',
'eliminate-local-multi-store',
'eliminate-local-single-block',
'eliminate-local-single-store',
'flatten-decorations',
'fold-spec-const-op-composite',
'freeze-spec-const',
'if-conversion',
'inline-entry-points-exhaustive',
'loop-fission',
'loop-fusion',
'loop-unroll',
'loop-unroll',
'loop-peeling',
'merge-blocks',
'merge-return',
'loop-unswitch',
'private-to-local',
'reduce-load-size',
'redundancy-elimination',
'remove-duplicates',
'replace-invalid-opcode',
'ssa-rewrite',
'scalar-replacement=100',
'scalar-replacement=42',
'strength-reduction',
'strip-debug',
'strip-reflect',
'vector-dce',
'workaround-1209',
'unify-const'
]
shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm')
output = placeholder.TempFileName('output.spv')
spirv_args = [shader, '-o', output, '--print-all'] + flags
expected_object_filenames = (output)
@inside_spirv_testsuite('SpirvOptFlags')
class TestPerformanceOptimizationPasses(expect.ValidObjectFile1_3,
expect.ExecutedListOfPasses):
"""Tests that spirv-opt schedules all the passes triggered by -O."""
flags = ['-O']
expected_passes = [
'merge-return',
'inline-entry-points-exhaustive',
'eliminate-dead-code-aggressive',
'private-to-local',
'eliminate-local-single-block',
'eliminate-local-single-store',
'eliminate-dead-code-aggressive',
'scalar-replacement=100',
'convert-local-access-chains',
'eliminate-local-single-block',
'eliminate-local-single-store',
'eliminate-dead-code-aggressive',
'eliminate-local-multi-store',
'eliminate-dead-code-aggressive',
'ccp',
'eliminate-dead-code-aggressive',
'redundancy-elimination',
'combine-access-chains',
'simplify-instructions',
'vector-dce',
'eliminate-dead-inserts',
'eliminate-dead-branches',
'simplify-instructions',
'if-conversion',
'copy-propagate-arrays',
'reduce-load-size',
'eliminate-dead-code-aggressive',
'merge-blocks',
'redundancy-elimination',
'eliminate-dead-branches',
'merge-blocks',
'simplify-instructions',
]
shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm')
output = placeholder.TempFileName('output.spv')
spirv_args = [shader, '-o', output, '--print-all'] + flags
expected_object_filenames = (output)
@inside_spirv_testsuite('SpirvOptFlags')
class TestSizeOptimizationPasses(expect.ValidObjectFile1_3,
expect.ExecutedListOfPasses):
"""Tests that spirv-opt schedules all the passes triggered by -Os."""
flags = ['-Os']
expected_passes = [
'merge-return',
'inline-entry-points-exhaustive',
'eliminate-dead-code-aggressive',
'private-to-local',
'scalar-replacement=100',
'convert-local-access-chains',
'eliminate-local-single-block',
'eliminate-local-single-store',
'eliminate-dead-code-aggressive',
'simplify-instructions',
'eliminate-dead-inserts',
'eliminate-local-multi-store',
'eliminate-dead-code-aggressive',
'ccp',
'eliminate-dead-code-aggressive',
'eliminate-dead-branches',
'if-conversion',
'eliminate-dead-code-aggressive',
'merge-blocks',
'simplify-instructions',
'eliminate-dead-inserts',
'redundancy-elimination',
'cfg-cleanup',
'eliminate-dead-code-aggressive',
]
shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm')
output = placeholder.TempFileName('output.spv')
spirv_args = [shader, '-o', output, '--print-all'] + flags
expected_object_filenames = (output)
@inside_spirv_testsuite('SpirvOptFlags')
class TestLegalizationPasses(expect.ValidObjectFile1_3,
expect.ExecutedListOfPasses):
"""Tests that spirv-opt schedules all the passes triggered by --legalize-hlsl.
"""
flags = ['--legalize-hlsl']
expected_passes = [
'eliminate-dead-branches',
'merge-return',
'inline-entry-points-exhaustive',
'eliminate-dead-functions',
'private-to-local',
'eliminate-local-single-block',
'eliminate-local-single-store',
'eliminate-dead-code-aggressive',
'scalar-replacement=0',
'eliminate-local-single-block',
'eliminate-local-single-store',
'eliminate-dead-code-aggressive',
'eliminate-local-multi-store',
'eliminate-dead-code-aggressive',
'ccp',
'eliminate-dead-branches',
'simplify-instructions',
'eliminate-dead-code-aggressive',
'copy-propagate-arrays',
'vector-dce',
'eliminate-dead-inserts',
'reduce-load-size',
'eliminate-dead-code-aggressive',
]
shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm')
output = placeholder.TempFileName('output.spv')
spirv_args = [shader, '-o', output, '--print-all'] + flags
expected_object_filenames = (output)
@inside_spirv_testsuite('SpirvOptFlags')
class TestScalarReplacementArgsNegative(expect.ErrorMessageSubstr):
"""Tests invalid arguments to --scalar-replacement."""
spirv_args = ['--scalar-replacement=-10']
expected_error_substr = 'must have no arguments or a positive integer argument'
@inside_spirv_testsuite('SpirvOptFlags')
class TestScalarReplacementArgsInvalidNumber(expect.ErrorMessageSubstr):
"""Tests invalid arguments to --scalar-replacement."""
spirv_args = ['--scalar-replacement=a10f']
expected_error_substr = 'must have no arguments or a positive integer argument'
@inside_spirv_testsuite('SpirvOptFlags')
class TestLoopFissionArgsNegative(expect.ErrorMessageSubstr):
"""Tests invalid arguments to --loop-fission."""
spirv_args = ['--loop-fission=-10']
expected_error_substr = 'must have a positive integer argument'
@inside_spirv_testsuite('SpirvOptFlags')
class TestLoopFissionArgsInvalidNumber(expect.ErrorMessageSubstr):
"""Tests invalid arguments to --loop-fission."""
spirv_args = ['--loop-fission=a10f']
expected_error_substr = 'must have a positive integer argument'
@inside_spirv_testsuite('SpirvOptFlags')
class TestLoopFusionArgsNegative(expect.ErrorMessageSubstr):
"""Tests invalid arguments to --loop-fusion."""
spirv_args = ['--loop-fusion=-10']
expected_error_substr = 'must have a positive integer argument'
@inside_spirv_testsuite('SpirvOptFlags')
class TestLoopFusionArgsInvalidNumber(expect.ErrorMessageSubstr):
"""Tests invalid arguments to --loop-fusion."""
spirv_args = ['--loop-fusion=a10f']
expected_error_substr = 'must have a positive integer argument'
@inside_spirv_testsuite('SpirvOptFlags')
class TestLoopUnrollPartialArgsNegative(expect.ErrorMessageSubstr):
"""Tests invalid arguments to --loop-unroll-partial."""
spirv_args = ['--loop-unroll-partial=-10']
expected_error_substr = 'must have a positive integer argument'
@inside_spirv_testsuite('SpirvOptFlags')
class TestLoopUnrollPartialArgsInvalidNumber(expect.ErrorMessageSubstr):
"""Tests invalid arguments to --loop-unroll-partial."""
spirv_args = ['--loop-unroll-partial=a10f']
expected_error_substr = 'must have a positive integer argument'
@inside_spirv_testsuite('SpirvOptFlags')
class TestLoopPeelingThresholdArgsNegative(expect.ErrorMessageSubstr):
"""Tests invalid arguments to --loop-peeling-threshold."""
spirv_args = ['--loop-peeling-threshold=-10']
expected_error_substr = 'must have a positive integer argument'
@inside_spirv_testsuite('SpirvOptFlags')
class TestLoopPeelingThresholdArgsInvalidNumber(expect.ErrorMessageSubstr):
"""Tests invalid arguments to --loop-peeling-threshold."""
spirv_args = ['--loop-peeling-threshold=a10f']
expected_error_substr = 'must have a positive integer argument'

58
test/tools/opt/oconfig.py Normal file
View File

@ -0,0 +1,58 @@
# Copyright (c) 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import placeholder
import expect
import re
from spirv_test_framework import inside_spirv_testsuite
def empty_main_assembly():
return """
OpCapability Shader
OpMemoryModel Logical GLSL450
OpEntryPoint Vertex %4 "main"
OpName %4 "main"
%2 = OpTypeVoid
%3 = OpTypeFunction %2
%4 = OpFunction %2 None %3
%5 = OpLabel
OpReturn
OpFunctionEnd"""
@inside_spirv_testsuite('SpirvOptConfigFile')
class TestOconfigEmpty(expect.SuccessfulReturn):
"""Tests empty config files are accepted."""
shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm')
config = placeholder.ConfigFlagsFile('', '.cfg')
spirv_args = [shader, '-o', placeholder.TempFileName('output.spv'), config]
@inside_spirv_testsuite('SpirvOptConfigFile')
class TestOconfigComments(expect.SuccessfulReturn):
"""Tests empty config files are accepted.
https://github.com/KhronosGroup/SPIRV-Tools/issues/1778
"""
shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm')
config = placeholder.ConfigFlagsFile("""
# This is a comment.
-O
--loop-unroll
""", '.cfg')
spirv_args = [shader, '-o', placeholder.TempFileName('output.spv'), config]

213
test/tools/placeholder.py Executable file
View File

@ -0,0 +1,213 @@
# Copyright (c) 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""A number of placeholders and their rules for expansion when used in tests.
These placeholders, when used in spirv_args or expected_* variables of
SpirvTest, have special meanings. In spirv_args, they will be substituted by
the result of instantiate_for_spirv_args(), while in expected_*, by
instantiate_for_expectation(). A TestCase instance will be passed in as
argument to the instantiate_*() methods.
"""
import os
import subprocess
import tempfile
from string import Template
class PlaceHolderException(Exception):
"""Exception class for PlaceHolder."""
pass
class PlaceHolder(object):
"""Base class for placeholders."""
def instantiate_for_spirv_args(self, testcase):
"""Instantiation rules for spirv_args.
This method will be called when the current placeholder appears in
spirv_args.
Returns:
A string to replace the current placeholder in spirv_args.
"""
raise PlaceHolderException('Subclass should implement this function.')
def instantiate_for_expectation(self, testcase):
"""Instantiation rules for expected_*.
This method will be called when the current placeholder appears in
expected_*.
Returns:
A string to replace the current placeholder in expected_*.
"""
raise PlaceHolderException('Subclass should implement this function.')
class FileShader(PlaceHolder):
"""Stands for a shader whose source code is in a file."""
def __init__(self, source, suffix, assembly_substr=None):
assert isinstance(source, str)
assert isinstance(suffix, str)
self.source = source
self.suffix = suffix
self.filename = None
# If provided, this is a substring which is expected to be in
# the disassembly of the module generated from this input file.
self.assembly_substr = assembly_substr
def instantiate_for_spirv_args(self, testcase):
"""Creates a temporary file and writes the source into it.
Returns:
The name of the temporary file.
"""
shader, self.filename = tempfile.mkstemp(
dir=testcase.directory, suffix=self.suffix)
shader_object = os.fdopen(shader, 'w')
shader_object.write(self.source)
shader_object.close()
return self.filename
def instantiate_for_expectation(self, testcase):
assert self.filename is not None
return self.filename
class ConfigFlagsFile(PlaceHolder):
"""Stands for a configuration file for spirv-opt generated out of a string."""
def __init__(self, content, suffix):
assert isinstance(content, str)
assert isinstance(suffix, str)
self.content = content
self.suffix = suffix
self.filename = None
def instantiate_for_spirv_args(self, testcase):
"""Creates a temporary file and writes content into it.
Returns:
The name of the temporary file.
"""
temp_fd, self.filename = tempfile.mkstemp(
dir=testcase.directory, suffix=self.suffix)
fd = os.fdopen(temp_fd, 'w')
fd.write(self.content)
fd.close()
return '-Oconfig=%s' % self.filename
def instantiate_for_expectation(self, testcase):
assert self.filename is not None
return self.filename
class FileSPIRVShader(PlaceHolder):
"""Stands for a source shader file which must be converted to SPIR-V."""
def __init__(self, source, suffix, assembly_substr=None):
assert isinstance(source, str)
assert isinstance(suffix, str)
self.source = source
self.suffix = suffix
self.filename = None
# If provided, this is a substring which is expected to be in
# the disassembly of the module generated from this input file.
self.assembly_substr = assembly_substr
def instantiate_for_spirv_args(self, testcase):
"""Creates a temporary file, writes the source into it and assembles it.
Returns:
The name of the assembled temporary file.
"""
shader, asm_filename = tempfile.mkstemp(
dir=testcase.directory, suffix=self.suffix)
shader_object = os.fdopen(shader, 'w')
shader_object.write(self.source)
shader_object.close()
self.filename = '%s.spv' % asm_filename
cmd = [
testcase.test_manager.assembler_path, asm_filename, '-o', self.filename
]
process = subprocess.Popen(
args=cmd,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=testcase.directory)
output = process.communicate()
assert process.returncode == 0 and not output[0] and not output[1]
return self.filename
def instantiate_for_expectation(self, testcase):
assert self.filename is not None
return self.filename
class StdinShader(PlaceHolder):
"""Stands for a shader whose source code is from stdin."""
def __init__(self, source):
assert isinstance(source, str)
self.source = source
self.filename = None
def instantiate_for_spirv_args(self, testcase):
"""Writes the source code back to the TestCase instance."""
testcase.stdin_shader = self.source
self.filename = '-'
return self.filename
def instantiate_for_expectation(self, testcase):
assert self.filename is not None
return self.filename
class TempFileName(PlaceHolder):
"""Stands for a temporary file's name."""
def __init__(self, filename):
assert isinstance(filename, str)
assert filename != ''
self.filename = filename
def instantiate_for_spirv_args(self, testcase):
return os.path.join(testcase.directory, self.filename)
def instantiate_for_expectation(self, testcase):
return os.path.join(testcase.directory, self.filename)
class SpecializedString(PlaceHolder):
"""Returns a string that has been specialized based on TestCase.
The string is specialized by expanding it as a string.Template
with all of the specialization being done with each $param replaced
by the associated member on TestCase.
"""
def __init__(self, filename):
assert isinstance(filename, str)
assert filename != ''
self.filename = filename
def instantiate_for_spirv_args(self, testcase):
return Template(self.filename).substitute(vars(testcase))
def instantiate_for_expectation(self, testcase):
return Template(self.filename).substitute(vars(testcase))

375
test/tools/spirv_test_framework.py Executable file
View File

@ -0,0 +1,375 @@
# Copyright (c) 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Manages and runs tests from the current working directory.
This will traverse the current working directory and look for python files that
contain subclasses of SpirvTest.
If a class has an @inside_spirv_testsuite decorator, an instance of that
class will be created and serve as a test case in that testsuite. The test
case is then run by the following steps:
1. A temporary directory will be created.
2. The spirv_args member variable will be inspected and all placeholders in it
will be expanded by calling instantiate_for_spirv_args() on placeholders.
The transformed list elements are then supplied as arguments to the spirv-*
tool under test.
3. If the environment member variable exists, its write() method will be
invoked.
4. All expected_* member variables will be inspected and all placeholders in
them will be expanded by calling instantiate_for_expectation() on those
placeholders. After placeholder expansion, if the expected_* variable is
a list, its element will be joined together with '' to form a single
string. These expected_* variables are to be used by the check_*() methods.
5. The spirv-* tool will be run with the arguments supplied in spirv_args.
6. All check_*() member methods will be called by supplying a TestStatus as
argument. Each check_*() method is expected to return a (Success, Message)
pair where Success is a boolean indicating success and Message is an error
message.
7. If any check_*() method fails, the error message is output and the
current test case fails.
If --leave-output was not specified, all temporary files and directories will
be deleted.
"""
from __future__ import print_function
import argparse
import fnmatch
import inspect
import os
import shutil
import subprocess
import sys
import tempfile
from collections import defaultdict
from placeholder import PlaceHolder
EXPECTED_BEHAVIOR_PREFIX = 'expected_'
VALIDATE_METHOD_PREFIX = 'check_'
def get_all_variables(instance):
"""Returns the names of all the variables in instance."""
return [v for v in dir(instance) if not callable(getattr(instance, v))]
def get_all_methods(instance):
"""Returns the names of all methods in instance."""
return [m for m in dir(instance) if callable(getattr(instance, m))]
def get_all_superclasses(cls):
"""Returns all superclasses of a given class.
Returns:
A list of superclasses of the given class. The order guarantees that
* A Base class precedes its derived classes, e.g., for "class B(A)", it
will be [..., A, B, ...].
* When there are multiple base classes, base classes declared first
precede those declared later, e.g., for "class C(A, B), it will be
[..., A, B, C, ...]
"""
classes = []
for superclass in cls.__bases__:
for c in get_all_superclasses(superclass):
if c not in classes:
classes.append(c)
for superclass in cls.__bases__:
if superclass not in classes:
classes.append(superclass)
return classes
def get_all_test_methods(test_class):
"""Gets all validation methods.
Returns:
A list of validation methods. The order guarantees that
* A method defined in superclass precedes one defined in subclass,
e.g., for "class A(B)", methods defined in B precedes those defined
in A.
* If a subclass has more than one superclass, e.g., "class C(A, B)",
then methods defined in A precedes those defined in B.
"""
classes = get_all_superclasses(test_class)
classes.append(test_class)
all_tests = [
m for c in classes for m in get_all_methods(c)
if m.startswith(VALIDATE_METHOD_PREFIX)
]
unique_tests = []
for t in all_tests:
if t not in unique_tests:
unique_tests.append(t)
return unique_tests
class SpirvTest:
"""Base class for spirv test cases.
Subclasses define test cases' facts (shader source code, spirv command,
result validation), which will be used by the TestCase class for running
tests. Subclasses should define spirv_args (specifying spirv_tool command
arguments), and at least one check_*() method (for result validation) for
a full-fledged test case. All check_*() methods should take a TestStatus
parameter and return a (Success, Message) pair, in which Success is a
boolean indicating success and Message is an error message. The test passes
iff all check_*() methods returns true.
Often, a test case class will delegate the check_* behaviors by inheriting
from other classes.
"""
def name(self):
return self.__class__.__name__
class TestStatus:
"""A struct for holding run status of a test case."""
def __init__(self, test_manager, returncode, stdout, stderr, directory,
inputs, input_filenames):
self.test_manager = test_manager
self.returncode = returncode
self.stdout = stdout
self.stderr = stderr
# temporary directory where the test runs
self.directory = directory
# List of inputs, as PlaceHolder objects.
self.inputs = inputs
# the names of input shader files (potentially including paths)
self.input_filenames = input_filenames
class SpirvTestException(Exception):
"""SpirvTest exception class."""
pass
def inside_spirv_testsuite(testsuite_name):
"""Decorator for subclasses of SpirvTest.
This decorator checks that a class meets the requirements (see below)
for a test case class, and then puts the class in a certain testsuite.
* The class needs to be a subclass of SpirvTest.
* The class needs to have spirv_args defined as a list.
* The class needs to define at least one check_*() methods.
* All expected_* variables required by check_*() methods can only be
of bool, str, or list type.
* Python runtime will throw an exception if the expected_* member
attributes required by check_*() methods are missing.
"""
def actual_decorator(cls):
if not inspect.isclass(cls):
raise SpirvTestException('Test case should be a class')
if not issubclass(cls, SpirvTest):
raise SpirvTestException(
'All test cases should be subclasses of SpirvTest')
if 'spirv_args' not in get_all_variables(cls):
raise SpirvTestException('No spirv_args found in the test case')
if not isinstance(cls.spirv_args, list):
raise SpirvTestException('spirv_args needs to be a list')
if not any(
[m.startswith(VALIDATE_METHOD_PREFIX) for m in get_all_methods(cls)]):
raise SpirvTestException('No check_*() methods found in the test case')
if not all(
[isinstance(v, (bool, str, list)) for v in get_all_variables(cls)]):
raise SpirvTestException(
'expected_* variables are only allowed to be bool, str, or '
'list type.')
cls.parent_testsuite = testsuite_name
return cls
return actual_decorator
class TestManager:
"""Manages and runs a set of tests."""
def __init__(self, executable_path, assembler_path, disassembler_path):
self.executable_path = executable_path
self.assembler_path = assembler_path
self.disassembler_path = disassembler_path
self.num_successes = 0
self.num_failures = 0
self.num_tests = 0
self.leave_output = False
self.tests = defaultdict(list)
def notify_result(self, test_case, success, message):
"""Call this to notify the manager of the results of a test run."""
self.num_successes += 1 if success else 0
self.num_failures += 0 if success else 1
counter_string = str(self.num_successes + self.num_failures) + '/' + str(
self.num_tests)
print('%-10s %-40s ' % (counter_string, test_case.test.name()) +
('Passed' if success else '-Failed-'))
if not success:
print(' '.join(test_case.command))
print(message)
def add_test(self, testsuite, test):
"""Add this to the current list of test cases."""
self.tests[testsuite].append(TestCase(test, self))
self.num_tests += 1
def run_tests(self):
for suite in self.tests:
print('SPIRV tool test suite: "{suite}"'.format(suite=suite))
for x in self.tests[suite]:
x.runTest()
class TestCase:
"""A single test case that runs in its own directory."""
def __init__(self, test, test_manager):
self.test = test
self.test_manager = test_manager
self.inputs = [] # inputs, as PlaceHolder objects.
self.file_shaders = [] # filenames of shader files.
self.stdin_shader = None # text to be passed to spirv_tool as stdin
def setUp(self):
"""Creates environment and instantiates placeholders for the test case."""
self.directory = tempfile.mkdtemp(dir=os.getcwd())
spirv_args = self.test.spirv_args
# Instantiate placeholders in spirv_args
self.test.spirv_args = [
arg.instantiate_for_spirv_args(self)
if isinstance(arg, PlaceHolder) else arg for arg in self.test.spirv_args
]
# Get all shader files' names
self.inputs = [arg for arg in spirv_args if isinstance(arg, PlaceHolder)]
self.file_shaders = [arg.filename for arg in self.inputs]
if 'environment' in get_all_variables(self.test):
self.test.environment.write(self.directory)
expectations = [
v for v in get_all_variables(self.test)
if v.startswith(EXPECTED_BEHAVIOR_PREFIX)
]
# Instantiate placeholders in expectations
for expectation_name in expectations:
expectation = getattr(self.test, expectation_name)
if isinstance(expectation, list):
expanded_expections = [
element.instantiate_for_expectation(self)
if isinstance(element, PlaceHolder) else element
for element in expectation
]
setattr(self.test, expectation_name, expanded_expections)
elif isinstance(expectation, PlaceHolder):
setattr(self.test, expectation_name,
expectation.instantiate_for_expectation(self))
def tearDown(self):
"""Removes the directory if we were not instructed to do otherwise."""
if not self.test_manager.leave_output:
shutil.rmtree(self.directory)
def runTest(self):
"""Sets up and runs a test, reports any failures and then cleans up."""
self.setUp()
success = False
message = ''
try:
self.command = [self.test_manager.executable_path]
self.command.extend(self.test.spirv_args)
process = subprocess.Popen(
args=self.command,
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=self.directory)
output = process.communicate(self.stdin_shader)
test_status = TestStatus(self.test_manager, process.returncode, output[0],
output[1], self.directory, self.inputs,
self.file_shaders)
run_results = [
getattr(self.test, test_method)(test_status)
for test_method in get_all_test_methods(self.test.__class__)
]
success, message = zip(*run_results)
success = all(success)
message = '\n'.join(message)
except Exception as e:
success = False
message = str(e)
self.test_manager.notify_result(
self, success,
message + '\nSTDOUT:\n%s\nSTDERR:\n%s' % (output[0], output[1]))
self.tearDown()
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'spirv_tool',
metavar='path/to/spirv_tool',
type=str,
nargs=1,
help='Path to the spirv-* tool under test')
parser.add_argument(
'spirv_as',
metavar='path/to/spirv-as',
type=str,
nargs=1,
help='Path to spirv-as')
parser.add_argument(
'spirv_dis',
metavar='path/to/spirv-dis',
type=str,
nargs=1,
help='Path to spirv-dis')
parser.add_argument(
'--leave-output',
action='store_const',
const=1,
help='Do not clean up temporary directories')
parser.add_argument(
'--test-dir', nargs=1, help='Directory to gather the tests from')
args = parser.parse_args()
default_path = sys.path
root_dir = os.getcwd()
if args.test_dir:
root_dir = args.test_dir[0]
manager = TestManager(args.spirv_tool[0], args.spirv_as[0], args.spirv_dis[0])
if args.leave_output:
manager.leave_output = True
for root, _, filenames in os.walk(root_dir):
for filename in fnmatch.filter(filenames, '*.py'):
if filename.endswith('nosetest.py'):
# Skip nose tests, which are for testing functions of
# the test framework.
continue
sys.path = default_path
sys.path.append(root)
mod = __import__(os.path.splitext(filename)[0])
for _, obj, in inspect.getmembers(mod):
if inspect.isclass(obj) and hasattr(obj, 'parent_testsuite'):
manager.add_test(obj.parent_testsuite, obj())
manager.run_tests()
if manager.num_failures > 0:
sys.exit(-1)
if __name__ == '__main__':
main()

155
test/tools/spirv_test_framework_nosetest.py Executable file
View File

@ -0,0 +1,155 @@
# Copyright (c) 2018 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from spirv_test_framework import get_all_test_methods, get_all_superclasses
from nose.tools import assert_equal, with_setup
# Classes to be used in testing get_all_{superclasses|test_methods}()
class Root:
def check_root(self):
pass
class A(Root):
def check_a(self):
pass
class B(Root):
def check_b(self):
pass
class C(Root):
def check_c(self):
pass
class D(Root):
def check_d(self):
pass
class E(Root):
def check_e(self):
pass
class H(B, C, D):
def check_h(self):
pass
class I(E):
def check_i(self):
pass
class O(H, I):
def check_o(self):
pass
class U(A, O):
def check_u(self):
pass
class X(U, A):
def check_x(self):
pass
class R1:
def check_r1(self):
pass
class R2:
def check_r2(self):
pass
class Multi(R1, R2):
def check_multi(self):
pass
def nosetest_get_all_superclasses():
"""Tests get_all_superclasses()."""
assert_equal(get_all_superclasses(A), [Root])
assert_equal(get_all_superclasses(B), [Root])
assert_equal(get_all_superclasses(C), [Root])
assert_equal(get_all_superclasses(D), [Root])
assert_equal(get_all_superclasses(E), [Root])
assert_equal(get_all_superclasses(H), [Root, B, C, D])
assert_equal(get_all_superclasses(I), [Root, E])
assert_equal(get_all_superclasses(O), [Root, B, C, D, E, H, I])
assert_equal(get_all_superclasses(U), [Root, B, C, D, E, H, I, A, O])
assert_equal(get_all_superclasses(X), [Root, B, C, D, E, H, I, A, O, U])
assert_equal(get_all_superclasses(Multi), [R1, R2])
def nosetest_get_all_methods():
"""Tests get_all_test_methods()."""
assert_equal(get_all_test_methods(A), ['check_root', 'check_a'])
assert_equal(get_all_test_methods(B), ['check_root', 'check_b'])
assert_equal(get_all_test_methods(C), ['check_root', 'check_c'])
assert_equal(get_all_test_methods(D), ['check_root', 'check_d'])
assert_equal(get_all_test_methods(E), ['check_root', 'check_e'])
assert_equal(
get_all_test_methods(H),
['check_root', 'check_b', 'check_c', 'check_d', 'check_h'])
assert_equal(get_all_test_methods(I), ['check_root', 'check_e', 'check_i'])
assert_equal(
get_all_test_methods(O), [
'check_root', 'check_b', 'check_c', 'check_d', 'check_e', 'check_h',
'check_i', 'check_o'
])
assert_equal(
get_all_test_methods(U), [
'check_root', 'check_b', 'check_c', 'check_d', 'check_e', 'check_h',
'check_i', 'check_a', 'check_o', 'check_u'
])
assert_equal(
get_all_test_methods(X), [
'check_root', 'check_b', 'check_c', 'check_d', 'check_e', 'check_h',
'check_i', 'check_a', 'check_o', 'check_u', 'check_x'
])
assert_equal(
get_all_test_methods(Multi), ['check_r1', 'check_r2', 'check_multi'])

20
tools/opt/opt.cpp
View File

@ -138,7 +138,7 @@ Options (in lexicographical order):
--eliminate-dead-functions
Deletes functions that cannot be reached from entry points or
exported functions.
--eliminate-dead-insert
--eliminate-dead-inserts
Deletes unreferenced inserts into composites, most notably
unused stores to vector components, that are not removed by
aggressive dead code elimination.
@ -383,10 +383,20 @@ bool ReadFlagsFromFile(const char* oconfig_flag,
return false;
}
while (!input_file.eof()) {
std::string flag;
input_file >> flag;
if (flag.length() > 0 && flag[0] != '#') {
std::string line;
while (std::getline(input_file, line)) {
// Ignore empty lines and lines starting with the comment marker '#'.
if (line.length() == 0 || line[0] == '#') {
continue;
}
// Tokenize the line. Add all found tokens to the list of found flags. This
// mimics the way the shell will parse whitespace on the command line. NOTE:
// This does not support quoting and it is not intended to.
std::istringstream iss(line);
while (!iss.eof()) {
std::string flag;
iss >> flag;
file_flags->push_back(flag);
}
}