From 03000a3a384fabf97da22d709e45e860aba86be1 Mon Sep 17 00:00:00 2001 From: Diego Novillo Date: Fri, 3 Aug 2018 18:08:46 -0400 Subject: [PATCH] Add testing framework for tools. This forks the testing harness from https://github.com/google/shaderc to allow testing CLI tools. New features needed for SPIRV-Tools include: 1- A new PlaceHolder subclass for spirv shaders. This place holder calls spirv-as to convert assembly input into SPIRV bytecode. This is required for most tools in SPIRV-Tools. 2- A minimal testing file for testing basic functionality of spirv-opt. Add tests for all flags in spirv-opt. 1. Adds tests to check that known flags match the names that each pass advertises. 2. Adds tests to check that -O, -Os and --legalize-hlsl schedule the expected passes. 3. Adds more functionality to Expect classes to support regular expression matching on stderr. 4. Add checks for integer arguments to optimization flags. 5. Fixes #1817 by modifying the parsing of integer arguments in flags that take them. 6. Fixes -Oconfig file parsing (#1778). It reads every line of the file into a string and then parses that string by tokenizing every group of characters between whitespaces (using the standard cin reading operator). This mimics shell command-line parsing, but it does not support quoting (and I'm not planning to). --- CMakeLists.txt | 1 + cmake/setup_build.cmake | 20 + source/opt/dead_variable_elimination.h | 2 +- source/opt/flatten_decoration_pass.h | 2 +- source/opt/loop_fission.h | 2 +- source/opt/loop_unroller.h | 2 +- source/opt/optimizer.cpp | 19 +- source/opt/replace_invalid_opc.h | 2 +- test/CMakeLists.txt | 1 + test/tools/CMakeLists.txt | 18 + test/tools/expect.py | 677 ++++++++++++++++++++ test/tools/expect_nosetest.py | 80 +++ test/tools/opt/CMakeLists.txt | 25 + test/tools/opt/flags.py | 330 ++++++++++ test/tools/opt/oconfig.py | 58 ++ test/tools/placeholder.py | 213 ++++++ test/tools/spirv_test_framework.py | 375 +++++++++++ test/tools/spirv_test_framework_nosetest.py | 155 +++++ tools/opt/opt.cpp | 20 +- 19 files changed, 1987 insertions(+), 15 deletions(-) create mode 100644 cmake/setup_build.cmake create mode 100644 test/tools/CMakeLists.txt create mode 100755 test/tools/expect.py create mode 100755 test/tools/expect_nosetest.py create mode 100644 test/tools/opt/CMakeLists.txt create mode 100644 test/tools/opt/flags.py create mode 100644 test/tools/opt/oconfig.py create mode 100755 test/tools/placeholder.py create mode 100755 test/tools/spirv_test_framework.py create mode 100755 test/tools/spirv_test_framework_nosetest.py diff --git a/CMakeLists.txt b/CMakeLists.txt index bcac38746..26e2d5ab2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,7 @@ enable_testing() set(SPIRV_TOOLS "SPIRV-Tools") include(GNUInstallDirs) +include(cmake/setup_build.cmake) set(CMAKE_POSITION_INDEPENDENT_CODE ON) diff --git a/cmake/setup_build.cmake b/cmake/setup_build.cmake new file mode 100644 index 000000000..6ba4c53d7 --- /dev/null +++ b/cmake/setup_build.cmake @@ -0,0 +1,20 @@ +# Find nosetests; see spirv_add_nosetests() for opting in to nosetests in a +# specific directory. +find_program(NOSETESTS_EXE NAMES nosetests PATHS $ENV{PYTHON_PACKAGE_PATH}) +if (NOT NOSETESTS_EXE) + message(STATUS "SPIRV-Tools: nosetests was not found - python support code will not be tested") +else() + message(STATUS "SPIRV-Tools: nosetests found - python support code will be tested") +endif() + +# Run nosetests on file ${PREFIX}_nosetest.py. Nosetests will look for classes +# and functions whose names start with "nosetest". The test name will be +# ${PREFIX}_nosetests. +function(spirv_add_nosetests PREFIX) + if(NOT "${SPIRV_SKIP_TESTS}" AND NOSETESTS_EXE) + add_test( + NAME ${PREFIX}_nosetests + COMMAND ${NOSETESTS_EXE} -m "^[Nn]ose[Tt]est" -v + ${CMAKE_CURRENT_SOURCE_DIR}/${PREFIX}_nosetest.py) + endif() +endfunction() diff --git a/source/opt/dead_variable_elimination.h b/source/opt/dead_variable_elimination.h index 052a754b9..40a7bc025 100644 --- a/source/opt/dead_variable_elimination.h +++ b/source/opt/dead_variable_elimination.h @@ -26,7 +26,7 @@ namespace opt { class DeadVariableElimination : public MemPass { public: - const char* name() const override { return "dead-variable-elimination"; } + const char* name() const override { return "eliminate-dead-variables"; } Status Process() override; IRContext::Analysis GetPreservedAnalyses() override { diff --git a/source/opt/flatten_decoration_pass.h b/source/opt/flatten_decoration_pass.h index c59821dc2..6a34f5bb2 100644 --- a/source/opt/flatten_decoration_pass.h +++ b/source/opt/flatten_decoration_pass.h @@ -25,7 +25,7 @@ namespace opt { // See optimizer.hpp for documentation. class FlattenDecorationPass : public Pass { public: - const char* name() const override { return "flatten-decoration"; } + const char* name() const override { return "flatten-decorations"; } Status Process() override; }; diff --git a/source/opt/loop_fission.h b/source/opt/loop_fission.h index ef886c965..e7a59c185 100644 --- a/source/opt/loop_fission.h +++ b/source/opt/loop_fission.h @@ -55,7 +55,7 @@ class LoopFissionPass : public Pass { bool split_multiple_times = true) : split_criteria_(functor), split_multiple_times_(split_multiple_times) {} - const char* name() const override { return "Loop Fission"; } + const char* name() const override { return "loop-fission"; } Pass::Status Process() override; diff --git a/source/opt/loop_unroller.h b/source/opt/loop_unroller.h index 98f4d475e..eb358ae24 100644 --- a/source/opt/loop_unroller.h +++ b/source/opt/loop_unroller.h @@ -26,7 +26,7 @@ class LoopUnroller : public Pass { LoopUnroller(bool fully_unroll, int unroll_factor) : Pass(), fully_unroll_(fully_unroll), unroll_factor_(unroll_factor) {} - const char* name() const override { return "Loop unroller"; } + const char* name() const override { return "loop-unroll"; } Status Process() override; diff --git a/source/opt/optimizer.cpp b/source/opt/optimizer.cpp index fc7d266e4..30654869c 100644 --- a/source/opt/optimizer.cpp +++ b/source/opt/optimizer.cpp @@ -119,7 +119,7 @@ Optimizer& Optimizer::RegisterLegalizationPasses() { .RegisterPass(CreateLocalSingleBlockLoadStoreElimPass()) .RegisterPass(CreateLocalSingleStoreElimPass()) .RegisterPass(CreateAggressiveDCEPass()) - // Split up aggragates so they are easier to deal with. + // Split up aggregates so they are easier to deal with. .RegisterPass(CreateScalarReplacementPass(0)) // Remove loads and stores so everything is in intermediate values. // Takes care of copy propagation of non-members. @@ -348,7 +348,7 @@ bool Optimizer::RegisterPassFromFlag(const std::string& flag) { if (pass_args.size() == 0) { RegisterPass(CreateScalarReplacementPass()); } else { - uint32_t limit = atoi(pass_args.c_str()); + int limit = atoi(pass_args.c_str()); if (limit > 0) { RegisterPass(CreateScalarReplacementPass(limit)); } else { @@ -409,7 +409,7 @@ bool Optimizer::RegisterPassFromFlag(const std::string& flag) { CreateLoopFusionPass(static_cast(max_registers_per_loop))); } else { Error(consumer(), nullptr, {}, - "--loop-fusion must be have a positive integer argument"); + "--loop-fusion must have a positive integer argument"); return false; } } else if (pass_name == "loop-unroll") { @@ -418,15 +418,24 @@ bool Optimizer::RegisterPassFromFlag(const std::string& flag) { RegisterPass(CreateVectorDCEPass()); } else if (pass_name == "loop-unroll-partial") { int factor = (pass_args.size() > 0) ? atoi(pass_args.c_str()) : 0; - if (factor != 0) { + if (factor > 0) { RegisterPass(CreateLoopUnrollPass(false, factor)); } else { Error(consumer(), nullptr, {}, - "--loop-unroll-partial must have a non-0 integer argument"); + "--loop-unroll-partial must have a positive integer argument"); return false; } } else if (pass_name == "loop-peeling") { RegisterPass(CreateLoopPeelingPass()); + } else if (pass_name == "loop-peeling-threshold") { + int factor = (pass_args.size() > 0) ? atoi(pass_args.c_str()) : 0; + if (factor > 0) { + opt::LoopPeelingPass::SetLoopPeelingThreshold(factor); + } else { + Error(consumer(), nullptr, {}, + "--loop-peeling-threshold must have a positive integer argument"); + return false; + } } else if (pass_name == "ccp") { RegisterPass(CreateCCPPass()); } else if (pass_name == "O") { diff --git a/source/opt/replace_invalid_opc.h b/source/opt/replace_invalid_opc.h index 4d4640582..426bcac5e 100644 --- a/source/opt/replace_invalid_opc.h +++ b/source/opt/replace_invalid_opc.h @@ -28,7 +28,7 @@ namespace opt { // value, the instruction will simply be deleted. class ReplaceInvalidOpcodePass : public Pass { public: - const char* name() const override { return "replace-invalid-opcodes"; } + const char* name() const override { return "replace-invalid-opcode"; } Status Process() override; private: diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 2d2e8b218..1fdf5a212 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -221,5 +221,6 @@ add_subdirectory(comp) add_subdirectory(link) add_subdirectory(opt) add_subdirectory(stats) +add_subdirectory(tools) add_subdirectory(util) add_subdirectory(val) diff --git a/test/tools/CMakeLists.txt b/test/tools/CMakeLists.txt new file mode 100644 index 000000000..cee95cadb --- /dev/null +++ b/test/tools/CMakeLists.txt @@ -0,0 +1,18 @@ +# Copyright (c) 2018 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +spirv_add_nosetests(expect) +spirv_add_nosetests(spirv_test_framework) + +add_subdirectory(opt) diff --git a/test/tools/expect.py b/test/tools/expect.py new file mode 100755 index 000000000..c9596506a --- /dev/null +++ b/test/tools/expect.py @@ -0,0 +1,677 @@ +# Copyright (c) 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A number of common spirv result checks coded in mixin classes. + +A test case can use these checks by declaring their enclosing mixin classes +as superclass and providing the expected_* variables required by the check_*() +methods in the mixin classes. +""" +import difflib +import os +import re +import subprocess +from spirv_test_framework import SpirvTest + + +def convert_to_unix_line_endings(source): + """Converts all line endings in source to be unix line endings.""" + return source.replace('\r\n', '\n').replace('\r', '\n') + + +def substitute_file_extension(filename, extension): + """Substitutes file extension, respecting known shader extensions. + + foo.vert -> foo.vert.[extension] [similarly for .frag, .comp, etc.] + foo.glsl -> foo.[extension] + foo.unknown -> foo.[extension] + foo -> foo.[extension] + """ + if filename[-5:] not in [ + '.vert', '.frag', '.tesc', '.tese', '.geom', '.comp', '.spvasm' + ]: + return filename.rsplit('.', 1)[0] + '.' + extension + else: + return filename + '.' + extension + + +def get_object_filename(source_filename): + """Gets the object filename for the given source file.""" + return substitute_file_extension(source_filename, 'spv') + + +def get_assembly_filename(source_filename): + """Gets the assembly filename for the given source file.""" + return substitute_file_extension(source_filename, 'spvasm') + + +def verify_file_non_empty(filename): + """Checks that a given file exists and is not empty.""" + if not os.path.isfile(filename): + return False, 'Cannot find file: ' + filename + if not os.path.getsize(filename): + return False, 'Empty file: ' + filename + return True, '' + + +class ReturnCodeIsZero(SpirvTest): + """Mixin class for checking that the return code is zero.""" + + def check_return_code_is_zero(self, status): + if status.returncode: + return False, 'Non-zero return code: {ret}\n'.format( + ret=status.returncode) + return True, '' + + +class NoOutputOnStdout(SpirvTest): + """Mixin class for checking that there is no output on stdout.""" + + def check_no_output_on_stdout(self, status): + if status.stdout: + return False, 'Non empty stdout: {out}\n'.format(out=status.stdout) + return True, '' + + +class NoOutputOnStderr(SpirvTest): + """Mixin class for checking that there is no output on stderr.""" + + def check_no_output_on_stderr(self, status): + if status.stderr: + return False, 'Non empty stderr: {err}\n'.format(err=status.stderr) + return True, '' + + +class SuccessfulReturn(ReturnCodeIsZero, NoOutputOnStdout, NoOutputOnStderr): + """Mixin class for checking that return code is zero and no output on + stdout and stderr.""" + pass + + +class NoGeneratedFiles(SpirvTest): + """Mixin class for checking that there is no file generated.""" + + def check_no_generated_files(self, status): + all_files = os.listdir(status.directory) + input_files = status.input_filenames + if all([f.startswith(status.directory) for f in input_files]): + all_files = [os.path.join(status.directory, f) for f in all_files] + generated_files = set(all_files) - set(input_files) + if len(generated_files) == 0: + return True, '' + else: + return False, 'Extra files generated: {}'.format(generated_files) + + +class CorrectBinaryLengthAndPreamble(SpirvTest): + """Provides methods for verifying preamble for a SPIR-V binary.""" + + def verify_binary_length_and_header(self, binary, spv_version=0x10000): + """Checks that the given SPIR-V binary has valid length and header. + + Returns: + False, error string if anything is invalid + True, '' otherwise + Args: + binary: a bytes object containing the SPIR-V binary + spv_version: target SPIR-V version number, with same encoding + as the version word in a SPIR-V header. + """ + + def read_word(binary, index, little_endian): + """Reads the index-th word from the given binary file.""" + word = binary[index * 4:(index + 1) * 4] + if little_endian: + word = reversed(word) + return reduce(lambda w, b: (w << 8) | ord(b), word, 0) + + def check_endianness(binary): + """Checks the endianness of the given SPIR-V binary. + + Returns: + True if it's little endian, False if it's big endian. + None if magic number is wrong. + """ + first_word = read_word(binary, 0, True) + if first_word == 0x07230203: + return True + first_word = read_word(binary, 0, False) + if first_word == 0x07230203: + return False + return None + + num_bytes = len(binary) + if num_bytes % 4 != 0: + return False, ('Incorrect SPV binary: size should be a multiple' + ' of words') + if num_bytes < 20: + return False, 'Incorrect SPV binary: size less than 5 words' + + preamble = binary[0:19] + little_endian = check_endianness(preamble) + # SPIR-V module magic number + if little_endian is None: + return False, 'Incorrect SPV binary: wrong magic number' + + # SPIR-V version number + version = read_word(preamble, 1, little_endian) + # TODO(dneto): Recent Glslang uses version word 0 for opengl_compat + # profile + + if version != spv_version and version != 0: + return False, 'Incorrect SPV binary: wrong version number' + # Shaderc-over-Glslang (0x000d....) or + # SPIRV-Tools (0x0007....) generator number + if read_word(preamble, 2, little_endian) != 0x000d0007 and \ + read_word(preamble, 2, little_endian) != 0x00070000: + return False, ('Incorrect SPV binary: wrong generator magic ' 'number') + # reserved for instruction schema + if read_word(preamble, 4, little_endian) != 0: + return False, 'Incorrect SPV binary: the 5th byte should be 0' + + return True, '' + + +class CorrectObjectFilePreamble(CorrectBinaryLengthAndPreamble): + """Provides methods for verifying preamble for a SPV object file.""" + + def verify_object_file_preamble(self, filename, spv_version=0x10000): + """Checks that the given SPIR-V binary file has correct preamble.""" + + success, message = verify_file_non_empty(filename) + if not success: + return False, message + + with open(filename, 'rb') as object_file: + object_file.seek(0, os.SEEK_END) + num_bytes = object_file.tell() + + object_file.seek(0) + + binary = bytes(object_file.read()) + return self.verify_binary_length_and_header(binary, spv_version) + + return True, '' + + +class CorrectAssemblyFilePreamble(SpirvTest): + """Provides methods for verifying preamble for a SPV assembly file.""" + + def verify_assembly_file_preamble(self, filename): + success, message = verify_file_non_empty(filename) + if not success: + return False, message + + with open(filename) as assembly_file: + line1 = assembly_file.readline() + line2 = assembly_file.readline() + line3 = assembly_file.readline() + + if (line1 != '; SPIR-V\n' or line2 != '; Version: 1.0\n' or + (not line3.startswith('; Generator: Google Shaderc over Glslang;'))): + return False, 'Incorrect SPV assembly' + + return True, '' + + +class ValidObjectFile(SuccessfulReturn, CorrectObjectFilePreamble): + """Mixin class for checking that every input file generates a valid SPIR-V 1.0 + object file following the object file naming rule, and there is no output on + stdout/stderr.""" + + def check_object_file_preamble(self, status): + for input_filename in status.input_filenames: + object_filename = get_object_filename(input_filename) + success, message = self.verify_object_file_preamble( + os.path.join(status.directory, object_filename)) + if not success: + return False, message + return True, '' + + +class ValidObjectFile1_3(ReturnCodeIsZero, CorrectObjectFilePreamble): + """Mixin class for checking that every input file generates a valid SPIR-V 1.3 + object file following the object file naming rule, and there is no output on + stdout/stderr.""" + + def check_object_file_preamble(self, status): + for input_filename in status.input_filenames: + object_filename = get_object_filename(input_filename) + success, message = self.verify_object_file_preamble( + os.path.join(status.directory, object_filename), 0x10300) + if not success: + return False, message + return True, '' + + +class ValidObjectFileWithAssemblySubstr(SuccessfulReturn, + CorrectObjectFilePreamble): + """Mixin class for checking that every input file generates a valid object + + file following the object file naming rule, there is no output on + stdout/stderr, and the disassmbly contains a specified substring per + input. + """ + + def check_object_file_disassembly(self, status): + for an_input in status.inputs: + object_filename = get_object_filename(an_input.filename) + obj_file = str(os.path.join(status.directory, object_filename)) + success, message = self.verify_object_file_preamble(obj_file) + if not success: + return False, message + cmd = [status.test_manager.disassembler_path, '--no-color', obj_file] + process = subprocess.Popen( + args=cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=status.directory) + output = process.communicate(None) + disassembly = output[0] + if not isinstance(an_input.assembly_substr, str): + return False, 'Missing assembly_substr member' + if an_input.assembly_substr not in disassembly: + return False, ('Incorrect disassembly output:\n{asm}\n' + 'Expected substring not found:\n{exp}'.format( + asm=disassembly, exp=an_input.assembly_substr)) + return True, '' + + +class ValidNamedObjectFile(SuccessfulReturn, CorrectObjectFilePreamble): + """Mixin class for checking that a list of object files with the given + names are correctly generated, and there is no output on stdout/stderr. + + To mix in this class, subclasses need to provide expected_object_filenames + as the expected object filenames. + """ + + def check_object_file_preamble(self, status): + for object_filename in self.expected_object_filenames: + success, message = self.verify_object_file_preamble( + os.path.join(status.directory, object_filename)) + if not success: + return False, message + return True, '' + + +class ValidFileContents(SpirvTest): + """Mixin class to test that a specific file contains specific text + To mix in this class, subclasses need to provide expected_file_contents as + the contents of the file and target_filename to determine the location.""" + + def check_file(self, status): + target_filename = os.path.join(status.directory, self.target_filename) + if not os.path.isfile(target_filename): + return False, 'Cannot find file: ' + target_filename + with open(target_filename, 'r') as target_file: + file_contents = target_file.read() + if isinstance(self.expected_file_contents, str): + if file_contents == self.expected_file_contents: + return True, '' + return False, ('Incorrect file output: \n{act}\n' + 'Expected:\n{exp}' + 'With diff:\n{diff}'.format( + act=file_contents, + exp=self.expected_file_contents, + diff='\n'.join( + list( + difflib.unified_diff( + self.expected_file_contents.split('\n'), + file_contents.split('\n'), + fromfile='expected_output', + tofile='actual_output'))))) + elif isinstance(self.expected_file_contents, type(re.compile(''))): + if self.expected_file_contents.search(file_contents): + return True, '' + return False, ('Incorrect file output: \n{act}\n' + 'Expected matching regex pattern:\n{exp}'.format( + act=file_contents, + exp=self.expected_file_contents.pattern)) + return False, ( + 'Could not open target file ' + target_filename + ' for reading') + + +class ValidAssemblyFile(SuccessfulReturn, CorrectAssemblyFilePreamble): + """Mixin class for checking that every input file generates a valid assembly + file following the assembly file naming rule, and there is no output on + stdout/stderr.""" + + def check_assembly_file_preamble(self, status): + for input_filename in status.input_filenames: + assembly_filename = get_assembly_filename(input_filename) + success, message = self.verify_assembly_file_preamble( + os.path.join(status.directory, assembly_filename)) + if not success: + return False, message + return True, '' + + +class ValidAssemblyFileWithSubstr(ValidAssemblyFile): + """Mixin class for checking that every input file generates a valid assembly + file following the assembly file naming rule, there is no output on + stdout/stderr, and all assembly files have the given substring specified + by expected_assembly_substr. + + To mix in this class, subclasses need to provde expected_assembly_substr + as the expected substring. + """ + + def check_assembly_with_substr(self, status): + for input_filename in status.input_filenames: + assembly_filename = get_assembly_filename(input_filename) + success, message = self.verify_assembly_file_preamble( + os.path.join(status.directory, assembly_filename)) + if not success: + return False, message + with open(assembly_filename, 'r') as f: + content = f.read() + if self.expected_assembly_substr not in convert_to_unix_line_endings( + content): + return False, ('Incorrect assembly output:\n{asm}\n' + 'Expected substring not found:\n{exp}'.format( + asm=content, exp=self.expected_assembly_substr)) + return True, '' + + +class ValidAssemblyFileWithoutSubstr(ValidAssemblyFile): + """Mixin class for checking that every input file generates a valid assembly + file following the assembly file naming rule, there is no output on + stdout/stderr, and no assembly files have the given substring specified + by unexpected_assembly_substr. + + To mix in this class, subclasses need to provde unexpected_assembly_substr + as the substring we expect not to see. + """ + + def check_assembly_for_substr(self, status): + for input_filename in status.input_filenames: + assembly_filename = get_assembly_filename(input_filename) + success, message = self.verify_assembly_file_preamble( + os.path.join(status.directory, assembly_filename)) + if not success: + return False, message + with open(assembly_filename, 'r') as f: + content = f.read() + if self.unexpected_assembly_substr in convert_to_unix_line_endings( + content): + return False, ('Incorrect assembly output:\n{asm}\n' + 'Unexpected substring found:\n{unexp}'.format( + asm=content, exp=self.unexpected_assembly_substr)) + return True, '' + + +class ValidNamedAssemblyFile(SuccessfulReturn, CorrectAssemblyFilePreamble): + """Mixin class for checking that a list of assembly files with the given + names are correctly generated, and there is no output on stdout/stderr. + + To mix in this class, subclasses need to provide expected_assembly_filenames + as the expected assembly filenames. + """ + + def check_object_file_preamble(self, status): + for assembly_filename in self.expected_assembly_filenames: + success, message = self.verify_assembly_file_preamble( + os.path.join(status.directory, assembly_filename)) + if not success: + return False, message + return True, '' + + +class ErrorMessage(SpirvTest): + """Mixin class for tests that fail with a specific error message. + + To mix in this class, subclasses need to provide expected_error as the + expected error message. + + The test should fail if the subprocess was terminated by a signal. + """ + + def check_has_error_message(self, status): + if not status.returncode: + return False, ('Expected error message, but returned success from ' + 'command execution') + if status.returncode < 0: + # On Unix, a negative value -N for Popen.returncode indicates + # termination by signal N. + # https://docs.python.org/2/library/subprocess.html + return False, ('Expected error message, but command was terminated by ' + 'signal ' + str(status.returncode)) + if not status.stderr: + return False, 'Expected error message, but no output on stderr' + if self.expected_error != convert_to_unix_line_endings(status.stderr): + return False, ('Incorrect stderr output:\n{act}\n' + 'Expected:\n{exp}'.format( + act=status.stderr, exp=self.expected_error)) + return True, '' + + +class ErrorMessageSubstr(SpirvTest): + """Mixin class for tests that fail with a specific substring in the error + message. + + To mix in this class, subclasses need to provide expected_error_substr as + the expected error message substring. + + The test should fail if the subprocess was terminated by a signal. + """ + + def check_has_error_message_as_substring(self, status): + if not status.returncode: + return False, ('Expected error message, but returned success from ' + 'command execution') + if status.returncode < 0: + # On Unix, a negative value -N for Popen.returncode indicates + # termination by signal N. + # https://docs.python.org/2/library/subprocess.html + return False, ('Expected error message, but command was terminated by ' + 'signal ' + str(status.returncode)) + if not status.stderr: + return False, 'Expected error message, but no output on stderr' + if self.expected_error_substr not in convert_to_unix_line_endings( + status.stderr): + return False, ('Incorrect stderr output:\n{act}\n' + 'Expected substring not found in stderr:\n{exp}'.format( + act=status.stderr, exp=self.expected_error_substr)) + return True, '' + + +class WarningMessage(SpirvTest): + """Mixin class for tests that succeed but have a specific warning message. + + To mix in this class, subclasses need to provide expected_warning as the + expected warning message. + """ + + def check_has_warning_message(self, status): + if status.returncode: + return False, ('Expected warning message, but returned failure from' + ' command execution') + if not status.stderr: + return False, 'Expected warning message, but no output on stderr' + if self.expected_warning != convert_to_unix_line_endings(status.stderr): + return False, ('Incorrect stderr output:\n{act}\n' + 'Expected:\n{exp}'.format( + act=status.stderr, exp=self.expected_warning)) + return True, '' + + +class ValidObjectFileWithWarning(NoOutputOnStdout, CorrectObjectFilePreamble, + WarningMessage): + """Mixin class for checking that every input file generates a valid object + file following the object file naming rule, with a specific warning message. + """ + + def check_object_file_preamble(self, status): + for input_filename in status.input_filenames: + object_filename = get_object_filename(input_filename) + success, message = self.verify_object_file_preamble( + os.path.join(status.directory, object_filename)) + if not success: + return False, message + return True, '' + + +class ValidAssemblyFileWithWarning(NoOutputOnStdout, + CorrectAssemblyFilePreamble, WarningMessage): + """Mixin class for checking that every input file generates a valid assembly + file following the assembly file naming rule, with a specific warning + message.""" + + def check_assembly_file_preamble(self, status): + for input_filename in status.input_filenames: + assembly_filename = get_assembly_filename(input_filename) + success, message = self.verify_assembly_file_preamble( + os.path.join(status.directory, assembly_filename)) + if not success: + return False, message + return True, '' + + +class StdoutMatch(SpirvTest): + """Mixin class for tests that can expect output on stdout. + + To mix in this class, subclasses need to provide expected_stdout as the + expected stdout output. + + For expected_stdout, if it's True, then they expect something on stdout but + will not check what it is. If it's a string, expect an exact match. If it's + anything else, it is assumed to be a compiled regular expression which will + be matched against re.search(). It will expect + expected_stdout.search(status.stdout) to be true. + """ + + def check_stdout_match(self, status): + # "True" in this case means we expect something on stdout, but we do not + # care what it is, we want to distinguish this from "blah" which means we + # expect exactly the string "blah". + if self.expected_stdout is True: + if not status.stdout: + return False, 'Expected something on stdout' + elif type(self.expected_stdout) == str: + if self.expected_stdout != convert_to_unix_line_endings(status.stdout): + return False, ('Incorrect stdout output:\n{ac}\n' + 'Expected:\n{ex}'.format( + ac=status.stdout, ex=self.expected_stdout)) + else: + if not self.expected_stdout.search( + convert_to_unix_line_endings(status.stdout)): + return False, ('Incorrect stdout output:\n{ac}\n' + 'Expected to match regex:\n{ex}'.format( + ac=status.stdout, ex=self.expected_stdout.pattern)) + return True, '' + + +class StderrMatch(SpirvTest): + """Mixin class for tests that can expect output on stderr. + + To mix in this class, subclasses need to provide expected_stderr as the + expected stderr output. + + For expected_stderr, if it's True, then they expect something on stderr, + but will not check what it is. If it's a string, expect an exact match. + If it's anything else, it is assumed to be a compiled regular expression + which will be matched against re.search(). It will expect + expected_stderr.search(status.stderr) to be true. + """ + + def check_stderr_match(self, status): + # "True" in this case means we expect something on stderr, but we do not + # care what it is, we want to distinguish this from "blah" which means we + # expect exactly the string "blah". + if self.expected_stderr is True: + if not status.stderr: + return False, 'Expected something on stderr' + elif type(self.expected_stderr) == str: + if self.expected_stderr != convert_to_unix_line_endings(status.stderr): + return False, ('Incorrect stderr output:\n{ac}\n' + 'Expected:\n{ex}'.format( + ac=status.stderr, ex=self.expected_stderr)) + else: + if not self.expected_stderr.search( + convert_to_unix_line_endings(status.stderr)): + return False, ('Incorrect stderr output:\n{ac}\n' + 'Expected to match regex:\n{ex}'.format( + ac=status.stderr, ex=self.expected_stderr.pattern)) + return True, '' + + +class StdoutNoWiderThan80Columns(SpirvTest): + """Mixin class for tests that require stdout to 80 characters or narrower. + + To mix in this class, subclasses need to provide expected_stdout as the + expected stdout output. + """ + + def check_stdout_not_too_wide(self, status): + if not status.stdout: + return True, '' + else: + for line in status.stdout.splitlines(): + if len(line) > 80: + return False, ('Stdout line longer than 80 columns: %s' % line) + return True, '' + + +class NoObjectFile(SpirvTest): + """Mixin class for checking that no input file has a corresponding object + file.""" + + def check_no_object_file(self, status): + for input_filename in status.input_filenames: + object_filename = get_object_filename(input_filename) + full_object_file = os.path.join(status.directory, object_filename) + print('checking %s' % full_object_file) + if os.path.isfile(full_object_file): + return False, ( + 'Expected no object file, but found: %s' % full_object_file) + return True, '' + + +class NoNamedOutputFiles(SpirvTest): + """Mixin class for checking that no specified output files exist. + + The expected_output_filenames member should be full pathnames.""" + + def check_no_named_output_files(self, status): + for object_filename in self.expected_output_filenames: + if os.path.isfile(object_filename): + return False, ( + 'Expected no output file, but found: %s' % object_filename) + return True, '' + + +class ExecutedListOfPasses(SpirvTest): + """Mixin class for checking that a list of passes where executed. + + It works by analyzing the output of the --print-all flag to spirv-opt. + + For this mixin to work, the class member expected_passes should be a sequence + of pass names as returned by Pass::name(). + """ + + def check_list_of_executed_passes(self, status): + # Collect all the output lines containing a pass name. + pass_names = [] + pass_name_re = re.compile(r'.*IR before pass (?P[\S]+)') + for line in status.stderr.splitlines(): + match = pass_name_re.match(line) + if match: + pass_names.append(match.group('pass_name')) + + for (expected, actual) in zip(self.expected_passes, pass_names): + if expected != actual: + return False, ( + 'Expected pass "%s" but found pass "%s"\n' % (expected, actual)) + + return True, '' diff --git a/test/tools/expect_nosetest.py b/test/tools/expect_nosetest.py new file mode 100755 index 000000000..b591a2d07 --- /dev/null +++ b/test/tools/expect_nosetest.py @@ -0,0 +1,80 @@ +# Copyright (c) 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Tests for the expect module.""" + +import expect +from spirv_test_framework import TestStatus +from nose.tools import assert_equal, assert_true, assert_false +import re + + +def nosetest_get_object_name(): + """Tests get_object_filename().""" + source_and_object_names = [('a.vert', 'a.vert.spv'), ('b.frag', 'b.frag.spv'), + ('c.tesc', 'c.tesc.spv'), ('d.tese', 'd.tese.spv'), + ('e.geom', 'e.geom.spv'), ('f.comp', 'f.comp.spv'), + ('file', 'file.spv'), ('file.', 'file.spv'), + ('file.uk', + 'file.spv'), ('file.vert.', + 'file.vert.spv'), ('file.vert.bla', + 'file.vert.spv')] + actual_object_names = [ + expect.get_object_filename(f[0]) for f in source_and_object_names + ] + expected_object_names = [f[1] for f in source_and_object_names] + + assert_equal(actual_object_names, expected_object_names) + + +class TestStdoutMatchADotC(expect.StdoutMatch): + expected_stdout = re.compile('a.c') + + +def nosetest_stdout_match_regex_has_match(): + test = TestStdoutMatchADotC() + status = TestStatus( + test_manager=None, + returncode=0, + stdout='0abc1', + stderr=None, + directory=None, + inputs=None, + input_filenames=None) + assert_true(test.check_stdout_match(status)[0]) + + +def nosetest_stdout_match_regex_no_match(): + test = TestStdoutMatchADotC() + status = TestStatus( + test_manager=None, + returncode=0, + stdout='ab', + stderr=None, + directory=None, + inputs=None, + input_filenames=None) + assert_false(test.check_stdout_match(status)[0]) + + +def nosetest_stdout_match_regex_empty_stdout(): + test = TestStdoutMatchADotC() + status = TestStatus( + test_manager=None, + returncode=0, + stdout='', + stderr=None, + directory=None, + inputs=None, + input_filenames=None) + assert_false(test.check_stdout_match(status)[0]) diff --git a/test/tools/opt/CMakeLists.txt b/test/tools/opt/CMakeLists.txt new file mode 100644 index 000000000..a6dc5262d --- /dev/null +++ b/test/tools/opt/CMakeLists.txt @@ -0,0 +1,25 @@ +# Copyright (c) 2018 Google LLC. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if(NOT ${SPIRV_SKIP_TESTS}) + if(${PYTHONINTERP_FOUND}) + add_test(NAME spirv_opt_tests + COMMAND ${PYTHON_EXECUTABLE} + ${CMAKE_CURRENT_SOURCE_DIR}/../spirv_test_framework.py + $ $ $ + --test-dir ${CMAKE_CURRENT_SOURCE_DIR}) + else() + message("Skipping CLI tools tests - Python executable not found") + endif() +endif() diff --git a/test/tools/opt/flags.py b/test/tools/opt/flags.py new file mode 100644 index 000000000..628d87108 --- /dev/null +++ b/test/tools/opt/flags.py @@ -0,0 +1,330 @@ +# Copyright (c) 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import placeholder +import expect +import re + +from spirv_test_framework import inside_spirv_testsuite + + +def empty_main_assembly(): + return """ + OpCapability Shader + OpMemoryModel Logical GLSL450 + OpEntryPoint Vertex %4 "main" + OpName %4 "main" + %2 = OpTypeVoid + %3 = OpTypeFunction %2 + %4 = OpFunction %2 None %3 + %5 = OpLabel + OpReturn + OpFunctionEnd""" + + +@inside_spirv_testsuite('SpirvOptBase') +class TestAssemblyFileAsOnlyParameter(expect.ValidObjectFile1_3): + """Tests that spirv-opt accepts a SPIR-V object file.""" + + shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm') + output = placeholder.TempFileName('output.spv') + spirv_args = [shader, '-o', output] + expected_object_filenames = (output) + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestHelpFlag(expect.ReturnCodeIsZero, expect.StdoutMatch): + """Test the --help flag.""" + + spirv_args = ['--help'] + expected_stdout = re.compile(r'.*The SPIR-V binary is read from ') + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestValidPassFlags(expect.ValidObjectFile1_3, + expect.ExecutedListOfPasses): + """Tests that spirv-opt accepts all valid optimization flags.""" + + flags = [ + '--ccp', '--cfg-cleanup', '--combine-access-chains', '--compact-ids', + '--convert-local-access-chains', '--copy-propagate-arrays', + '--eliminate-common-uniform', '--eliminate-dead-branches', + '--eliminate-dead-code-aggressive', '--eliminate-dead-const', + '--eliminate-dead-functions', '--eliminate-dead-inserts', + '--eliminate-dead-variables', '--eliminate-insert-extract', + '--eliminate-local-multi-store', '--eliminate-local-single-block', + '--eliminate-local-single-store', '--flatten-decorations', + '--fold-spec-const-op-composite', '--freeze-spec-const', + '--if-conversion', '--inline-entry-points-exhaustive', '--loop-fission', + '20', '--loop-fusion', '5', '--loop-unroll', '--loop-unroll-partial', '3', + '--loop-peeling', '--merge-blocks', '--merge-return', '--loop-unswitch', + '--private-to-local', '--reduce-load-size', '--redundancy-elimination', + '--remove-duplicates', '--replace-invalid-opcode', '--ssa-rewrite', + '--scalar-replacement', '--scalar-replacement=42', '--strength-reduction', + '--strip-debug', '--strip-reflect', '--vector-dce', '--workaround-1209', + '--unify-const' + ] + expected_passes = [ + 'ccp', + 'cfg-cleanup', + 'combine-access-chains', + 'compact-ids', + 'convert-local-access-chains', + 'copy-propagate-arrays', + 'eliminate-common-uniform', + 'eliminate-dead-branches', + 'eliminate-dead-code-aggressive', + 'eliminate-dead-const', + 'eliminate-dead-functions', + 'eliminate-dead-inserts', + 'eliminate-dead-variables', + # --eliminate-insert-extract runs the simplify-instructions pass. + 'simplify-instructions', + 'eliminate-local-multi-store', + 'eliminate-local-single-block', + 'eliminate-local-single-store', + 'flatten-decorations', + 'fold-spec-const-op-composite', + 'freeze-spec-const', + 'if-conversion', + 'inline-entry-points-exhaustive', + 'loop-fission', + 'loop-fusion', + 'loop-unroll', + 'loop-unroll', + 'loop-peeling', + 'merge-blocks', + 'merge-return', + 'loop-unswitch', + 'private-to-local', + 'reduce-load-size', + 'redundancy-elimination', + 'remove-duplicates', + 'replace-invalid-opcode', + 'ssa-rewrite', + 'scalar-replacement=100', + 'scalar-replacement=42', + 'strength-reduction', + 'strip-debug', + 'strip-reflect', + 'vector-dce', + 'workaround-1209', + 'unify-const' + ] + shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm') + output = placeholder.TempFileName('output.spv') + spirv_args = [shader, '-o', output, '--print-all'] + flags + expected_object_filenames = (output) + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestPerformanceOptimizationPasses(expect.ValidObjectFile1_3, + expect.ExecutedListOfPasses): + """Tests that spirv-opt schedules all the passes triggered by -O.""" + + flags = ['-O'] + expected_passes = [ + 'merge-return', + 'inline-entry-points-exhaustive', + 'eliminate-dead-code-aggressive', + 'private-to-local', + 'eliminate-local-single-block', + 'eliminate-local-single-store', + 'eliminate-dead-code-aggressive', + 'scalar-replacement=100', + 'convert-local-access-chains', + 'eliminate-local-single-block', + 'eliminate-local-single-store', + 'eliminate-dead-code-aggressive', + 'eliminate-local-multi-store', + 'eliminate-dead-code-aggressive', + 'ccp', + 'eliminate-dead-code-aggressive', + 'redundancy-elimination', + 'combine-access-chains', + 'simplify-instructions', + 'vector-dce', + 'eliminate-dead-inserts', + 'eliminate-dead-branches', + 'simplify-instructions', + 'if-conversion', + 'copy-propagate-arrays', + 'reduce-load-size', + 'eliminate-dead-code-aggressive', + 'merge-blocks', + 'redundancy-elimination', + 'eliminate-dead-branches', + 'merge-blocks', + 'simplify-instructions', + ] + shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm') + output = placeholder.TempFileName('output.spv') + spirv_args = [shader, '-o', output, '--print-all'] + flags + expected_object_filenames = (output) + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestSizeOptimizationPasses(expect.ValidObjectFile1_3, + expect.ExecutedListOfPasses): + """Tests that spirv-opt schedules all the passes triggered by -Os.""" + + flags = ['-Os'] + expected_passes = [ + 'merge-return', + 'inline-entry-points-exhaustive', + 'eliminate-dead-code-aggressive', + 'private-to-local', + 'scalar-replacement=100', + 'convert-local-access-chains', + 'eliminate-local-single-block', + 'eliminate-local-single-store', + 'eliminate-dead-code-aggressive', + 'simplify-instructions', + 'eliminate-dead-inserts', + 'eliminate-local-multi-store', + 'eliminate-dead-code-aggressive', + 'ccp', + 'eliminate-dead-code-aggressive', + 'eliminate-dead-branches', + 'if-conversion', + 'eliminate-dead-code-aggressive', + 'merge-blocks', + 'simplify-instructions', + 'eliminate-dead-inserts', + 'redundancy-elimination', + 'cfg-cleanup', + 'eliminate-dead-code-aggressive', + ] + shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm') + output = placeholder.TempFileName('output.spv') + spirv_args = [shader, '-o', output, '--print-all'] + flags + expected_object_filenames = (output) + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestLegalizationPasses(expect.ValidObjectFile1_3, + expect.ExecutedListOfPasses): + """Tests that spirv-opt schedules all the passes triggered by --legalize-hlsl. + """ + + flags = ['--legalize-hlsl'] + expected_passes = [ + 'eliminate-dead-branches', + 'merge-return', + 'inline-entry-points-exhaustive', + 'eliminate-dead-functions', + 'private-to-local', + 'eliminate-local-single-block', + 'eliminate-local-single-store', + 'eliminate-dead-code-aggressive', + 'scalar-replacement=0', + 'eliminate-local-single-block', + 'eliminate-local-single-store', + 'eliminate-dead-code-aggressive', + 'eliminate-local-multi-store', + 'eliminate-dead-code-aggressive', + 'ccp', + 'eliminate-dead-branches', + 'simplify-instructions', + 'eliminate-dead-code-aggressive', + 'copy-propagate-arrays', + 'vector-dce', + 'eliminate-dead-inserts', + 'reduce-load-size', + 'eliminate-dead-code-aggressive', + ] + shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm') + output = placeholder.TempFileName('output.spv') + spirv_args = [shader, '-o', output, '--print-all'] + flags + expected_object_filenames = (output) + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestScalarReplacementArgsNegative(expect.ErrorMessageSubstr): + """Tests invalid arguments to --scalar-replacement.""" + + spirv_args = ['--scalar-replacement=-10'] + expected_error_substr = 'must have no arguments or a positive integer argument' + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestScalarReplacementArgsInvalidNumber(expect.ErrorMessageSubstr): + """Tests invalid arguments to --scalar-replacement.""" + + spirv_args = ['--scalar-replacement=a10f'] + expected_error_substr = 'must have no arguments or a positive integer argument' + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestLoopFissionArgsNegative(expect.ErrorMessageSubstr): + """Tests invalid arguments to --loop-fission.""" + + spirv_args = ['--loop-fission=-10'] + expected_error_substr = 'must have a positive integer argument' + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestLoopFissionArgsInvalidNumber(expect.ErrorMessageSubstr): + """Tests invalid arguments to --loop-fission.""" + + spirv_args = ['--loop-fission=a10f'] + expected_error_substr = 'must have a positive integer argument' + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestLoopFusionArgsNegative(expect.ErrorMessageSubstr): + """Tests invalid arguments to --loop-fusion.""" + + spirv_args = ['--loop-fusion=-10'] + expected_error_substr = 'must have a positive integer argument' + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestLoopFusionArgsInvalidNumber(expect.ErrorMessageSubstr): + """Tests invalid arguments to --loop-fusion.""" + + spirv_args = ['--loop-fusion=a10f'] + expected_error_substr = 'must have a positive integer argument' + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestLoopUnrollPartialArgsNegative(expect.ErrorMessageSubstr): + """Tests invalid arguments to --loop-unroll-partial.""" + + spirv_args = ['--loop-unroll-partial=-10'] + expected_error_substr = 'must have a positive integer argument' + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestLoopUnrollPartialArgsInvalidNumber(expect.ErrorMessageSubstr): + """Tests invalid arguments to --loop-unroll-partial.""" + + spirv_args = ['--loop-unroll-partial=a10f'] + expected_error_substr = 'must have a positive integer argument' + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestLoopPeelingThresholdArgsNegative(expect.ErrorMessageSubstr): + """Tests invalid arguments to --loop-peeling-threshold.""" + + spirv_args = ['--loop-peeling-threshold=-10'] + expected_error_substr = 'must have a positive integer argument' + + +@inside_spirv_testsuite('SpirvOptFlags') +class TestLoopPeelingThresholdArgsInvalidNumber(expect.ErrorMessageSubstr): + """Tests invalid arguments to --loop-peeling-threshold.""" + + spirv_args = ['--loop-peeling-threshold=a10f'] + expected_error_substr = 'must have a positive integer argument' diff --git a/test/tools/opt/oconfig.py b/test/tools/opt/oconfig.py new file mode 100644 index 000000000..337237994 --- /dev/null +++ b/test/tools/opt/oconfig.py @@ -0,0 +1,58 @@ +# Copyright (c) 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import placeholder +import expect +import re + +from spirv_test_framework import inside_spirv_testsuite + + +def empty_main_assembly(): + return """ + OpCapability Shader + OpMemoryModel Logical GLSL450 + OpEntryPoint Vertex %4 "main" + OpName %4 "main" + %2 = OpTypeVoid + %3 = OpTypeFunction %2 + %4 = OpFunction %2 None %3 + %5 = OpLabel + OpReturn + OpFunctionEnd""" + + +@inside_spirv_testsuite('SpirvOptConfigFile') +class TestOconfigEmpty(expect.SuccessfulReturn): + """Tests empty config files are accepted.""" + + shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm') + config = placeholder.ConfigFlagsFile('', '.cfg') + spirv_args = [shader, '-o', placeholder.TempFileName('output.spv'), config] + + +@inside_spirv_testsuite('SpirvOptConfigFile') +class TestOconfigComments(expect.SuccessfulReturn): + """Tests empty config files are accepted. + + https://github.com/KhronosGroup/SPIRV-Tools/issues/1778 + """ + + shader = placeholder.FileSPIRVShader(empty_main_assembly(), '.spvasm') + config = placeholder.ConfigFlagsFile(""" +# This is a comment. +-O +--loop-unroll +""", '.cfg') + spirv_args = [shader, '-o', placeholder.TempFileName('output.spv'), config] diff --git a/test/tools/placeholder.py b/test/tools/placeholder.py new file mode 100755 index 000000000..7de3c467a --- /dev/null +++ b/test/tools/placeholder.py @@ -0,0 +1,213 @@ +# Copyright (c) 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""A number of placeholders and their rules for expansion when used in tests. + +These placeholders, when used in spirv_args or expected_* variables of +SpirvTest, have special meanings. In spirv_args, they will be substituted by +the result of instantiate_for_spirv_args(), while in expected_*, by +instantiate_for_expectation(). A TestCase instance will be passed in as +argument to the instantiate_*() methods. +""" + +import os +import subprocess +import tempfile +from string import Template + + +class PlaceHolderException(Exception): + """Exception class for PlaceHolder.""" + pass + + +class PlaceHolder(object): + """Base class for placeholders.""" + + def instantiate_for_spirv_args(self, testcase): + """Instantiation rules for spirv_args. + + This method will be called when the current placeholder appears in + spirv_args. + + Returns: + A string to replace the current placeholder in spirv_args. + """ + raise PlaceHolderException('Subclass should implement this function.') + + def instantiate_for_expectation(self, testcase): + """Instantiation rules for expected_*. + + This method will be called when the current placeholder appears in + expected_*. + + Returns: + A string to replace the current placeholder in expected_*. + """ + raise PlaceHolderException('Subclass should implement this function.') + + +class FileShader(PlaceHolder): + """Stands for a shader whose source code is in a file.""" + + def __init__(self, source, suffix, assembly_substr=None): + assert isinstance(source, str) + assert isinstance(suffix, str) + self.source = source + self.suffix = suffix + self.filename = None + # If provided, this is a substring which is expected to be in + # the disassembly of the module generated from this input file. + self.assembly_substr = assembly_substr + + def instantiate_for_spirv_args(self, testcase): + """Creates a temporary file and writes the source into it. + + Returns: + The name of the temporary file. + """ + shader, self.filename = tempfile.mkstemp( + dir=testcase.directory, suffix=self.suffix) + shader_object = os.fdopen(shader, 'w') + shader_object.write(self.source) + shader_object.close() + return self.filename + + def instantiate_for_expectation(self, testcase): + assert self.filename is not None + return self.filename + + +class ConfigFlagsFile(PlaceHolder): + """Stands for a configuration file for spirv-opt generated out of a string.""" + + def __init__(self, content, suffix): + assert isinstance(content, str) + assert isinstance(suffix, str) + self.content = content + self.suffix = suffix + self.filename = None + + def instantiate_for_spirv_args(self, testcase): + """Creates a temporary file and writes content into it. + + Returns: + The name of the temporary file. + """ + temp_fd, self.filename = tempfile.mkstemp( + dir=testcase.directory, suffix=self.suffix) + fd = os.fdopen(temp_fd, 'w') + fd.write(self.content) + fd.close() + return '-Oconfig=%s' % self.filename + + def instantiate_for_expectation(self, testcase): + assert self.filename is not None + return self.filename + + +class FileSPIRVShader(PlaceHolder): + """Stands for a source shader file which must be converted to SPIR-V.""" + + def __init__(self, source, suffix, assembly_substr=None): + assert isinstance(source, str) + assert isinstance(suffix, str) + self.source = source + self.suffix = suffix + self.filename = None + # If provided, this is a substring which is expected to be in + # the disassembly of the module generated from this input file. + self.assembly_substr = assembly_substr + + def instantiate_for_spirv_args(self, testcase): + """Creates a temporary file, writes the source into it and assembles it. + + Returns: + The name of the assembled temporary file. + """ + shader, asm_filename = tempfile.mkstemp( + dir=testcase.directory, suffix=self.suffix) + shader_object = os.fdopen(shader, 'w') + shader_object.write(self.source) + shader_object.close() + self.filename = '%s.spv' % asm_filename + cmd = [ + testcase.test_manager.assembler_path, asm_filename, '-o', self.filename + ] + process = subprocess.Popen( + args=cmd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=testcase.directory) + output = process.communicate() + assert process.returncode == 0 and not output[0] and not output[1] + return self.filename + + def instantiate_for_expectation(self, testcase): + assert self.filename is not None + return self.filename + + +class StdinShader(PlaceHolder): + """Stands for a shader whose source code is from stdin.""" + + def __init__(self, source): + assert isinstance(source, str) + self.source = source + self.filename = None + + def instantiate_for_spirv_args(self, testcase): + """Writes the source code back to the TestCase instance.""" + testcase.stdin_shader = self.source + self.filename = '-' + return self.filename + + def instantiate_for_expectation(self, testcase): + assert self.filename is not None + return self.filename + + +class TempFileName(PlaceHolder): + """Stands for a temporary file's name.""" + + def __init__(self, filename): + assert isinstance(filename, str) + assert filename != '' + self.filename = filename + + def instantiate_for_spirv_args(self, testcase): + return os.path.join(testcase.directory, self.filename) + + def instantiate_for_expectation(self, testcase): + return os.path.join(testcase.directory, self.filename) + + +class SpecializedString(PlaceHolder): + """Returns a string that has been specialized based on TestCase. + + The string is specialized by expanding it as a string.Template + with all of the specialization being done with each $param replaced + by the associated member on TestCase. + """ + + def __init__(self, filename): + assert isinstance(filename, str) + assert filename != '' + self.filename = filename + + def instantiate_for_spirv_args(self, testcase): + return Template(self.filename).substitute(vars(testcase)) + + def instantiate_for_expectation(self, testcase): + return Template(self.filename).substitute(vars(testcase)) diff --git a/test/tools/spirv_test_framework.py b/test/tools/spirv_test_framework.py new file mode 100755 index 000000000..03ad08fa8 --- /dev/null +++ b/test/tools/spirv_test_framework.py @@ -0,0 +1,375 @@ +# Copyright (c) 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Manages and runs tests from the current working directory. + +This will traverse the current working directory and look for python files that +contain subclasses of SpirvTest. + +If a class has an @inside_spirv_testsuite decorator, an instance of that +class will be created and serve as a test case in that testsuite. The test +case is then run by the following steps: + + 1. A temporary directory will be created. + 2. The spirv_args member variable will be inspected and all placeholders in it + will be expanded by calling instantiate_for_spirv_args() on placeholders. + The transformed list elements are then supplied as arguments to the spirv-* + tool under test. + 3. If the environment member variable exists, its write() method will be + invoked. + 4. All expected_* member variables will be inspected and all placeholders in + them will be expanded by calling instantiate_for_expectation() on those + placeholders. After placeholder expansion, if the expected_* variable is + a list, its element will be joined together with '' to form a single + string. These expected_* variables are to be used by the check_*() methods. + 5. The spirv-* tool will be run with the arguments supplied in spirv_args. + 6. All check_*() member methods will be called by supplying a TestStatus as + argument. Each check_*() method is expected to return a (Success, Message) + pair where Success is a boolean indicating success and Message is an error + message. + 7. If any check_*() method fails, the error message is output and the + current test case fails. + +If --leave-output was not specified, all temporary files and directories will +be deleted. +""" + +from __future__ import print_function + +import argparse +import fnmatch +import inspect +import os +import shutil +import subprocess +import sys +import tempfile +from collections import defaultdict +from placeholder import PlaceHolder + +EXPECTED_BEHAVIOR_PREFIX = 'expected_' +VALIDATE_METHOD_PREFIX = 'check_' + + +def get_all_variables(instance): + """Returns the names of all the variables in instance.""" + return [v for v in dir(instance) if not callable(getattr(instance, v))] + + +def get_all_methods(instance): + """Returns the names of all methods in instance.""" + return [m for m in dir(instance) if callable(getattr(instance, m))] + + +def get_all_superclasses(cls): + """Returns all superclasses of a given class. + + Returns: + A list of superclasses of the given class. The order guarantees that + * A Base class precedes its derived classes, e.g., for "class B(A)", it + will be [..., A, B, ...]. + * When there are multiple base classes, base classes declared first + precede those declared later, e.g., for "class C(A, B), it will be + [..., A, B, C, ...] + """ + classes = [] + for superclass in cls.__bases__: + for c in get_all_superclasses(superclass): + if c not in classes: + classes.append(c) + for superclass in cls.__bases__: + if superclass not in classes: + classes.append(superclass) + return classes + + +def get_all_test_methods(test_class): + """Gets all validation methods. + + Returns: + A list of validation methods. The order guarantees that + * A method defined in superclass precedes one defined in subclass, + e.g., for "class A(B)", methods defined in B precedes those defined + in A. + * If a subclass has more than one superclass, e.g., "class C(A, B)", + then methods defined in A precedes those defined in B. + """ + classes = get_all_superclasses(test_class) + classes.append(test_class) + all_tests = [ + m for c in classes for m in get_all_methods(c) + if m.startswith(VALIDATE_METHOD_PREFIX) + ] + unique_tests = [] + for t in all_tests: + if t not in unique_tests: + unique_tests.append(t) + return unique_tests + + +class SpirvTest: + """Base class for spirv test cases. + + Subclasses define test cases' facts (shader source code, spirv command, + result validation), which will be used by the TestCase class for running + tests. Subclasses should define spirv_args (specifying spirv_tool command + arguments), and at least one check_*() method (for result validation) for + a full-fledged test case. All check_*() methods should take a TestStatus + parameter and return a (Success, Message) pair, in which Success is a + boolean indicating success and Message is an error message. The test passes + iff all check_*() methods returns true. + + Often, a test case class will delegate the check_* behaviors by inheriting + from other classes. + """ + + def name(self): + return self.__class__.__name__ + + +class TestStatus: + """A struct for holding run status of a test case.""" + + def __init__(self, test_manager, returncode, stdout, stderr, directory, + inputs, input_filenames): + self.test_manager = test_manager + self.returncode = returncode + self.stdout = stdout + self.stderr = stderr + # temporary directory where the test runs + self.directory = directory + # List of inputs, as PlaceHolder objects. + self.inputs = inputs + # the names of input shader files (potentially including paths) + self.input_filenames = input_filenames + + +class SpirvTestException(Exception): + """SpirvTest exception class.""" + pass + + +def inside_spirv_testsuite(testsuite_name): + """Decorator for subclasses of SpirvTest. + + This decorator checks that a class meets the requirements (see below) + for a test case class, and then puts the class in a certain testsuite. + * The class needs to be a subclass of SpirvTest. + * The class needs to have spirv_args defined as a list. + * The class needs to define at least one check_*() methods. + * All expected_* variables required by check_*() methods can only be + of bool, str, or list type. + * Python runtime will throw an exception if the expected_* member + attributes required by check_*() methods are missing. + """ + + def actual_decorator(cls): + if not inspect.isclass(cls): + raise SpirvTestException('Test case should be a class') + if not issubclass(cls, SpirvTest): + raise SpirvTestException( + 'All test cases should be subclasses of SpirvTest') + if 'spirv_args' not in get_all_variables(cls): + raise SpirvTestException('No spirv_args found in the test case') + if not isinstance(cls.spirv_args, list): + raise SpirvTestException('spirv_args needs to be a list') + if not any( + [m.startswith(VALIDATE_METHOD_PREFIX) for m in get_all_methods(cls)]): + raise SpirvTestException('No check_*() methods found in the test case') + if not all( + [isinstance(v, (bool, str, list)) for v in get_all_variables(cls)]): + raise SpirvTestException( + 'expected_* variables are only allowed to be bool, str, or ' + 'list type.') + cls.parent_testsuite = testsuite_name + return cls + + return actual_decorator + + +class TestManager: + """Manages and runs a set of tests.""" + + def __init__(self, executable_path, assembler_path, disassembler_path): + self.executable_path = executable_path + self.assembler_path = assembler_path + self.disassembler_path = disassembler_path + self.num_successes = 0 + self.num_failures = 0 + self.num_tests = 0 + self.leave_output = False + self.tests = defaultdict(list) + + def notify_result(self, test_case, success, message): + """Call this to notify the manager of the results of a test run.""" + self.num_successes += 1 if success else 0 + self.num_failures += 0 if success else 1 + counter_string = str(self.num_successes + self.num_failures) + '/' + str( + self.num_tests) + print('%-10s %-40s ' % (counter_string, test_case.test.name()) + + ('Passed' if success else '-Failed-')) + if not success: + print(' '.join(test_case.command)) + print(message) + + def add_test(self, testsuite, test): + """Add this to the current list of test cases.""" + self.tests[testsuite].append(TestCase(test, self)) + self.num_tests += 1 + + def run_tests(self): + for suite in self.tests: + print('SPIRV tool test suite: "{suite}"'.format(suite=suite)) + for x in self.tests[suite]: + x.runTest() + + +class TestCase: + """A single test case that runs in its own directory.""" + + def __init__(self, test, test_manager): + self.test = test + self.test_manager = test_manager + self.inputs = [] # inputs, as PlaceHolder objects. + self.file_shaders = [] # filenames of shader files. + self.stdin_shader = None # text to be passed to spirv_tool as stdin + + def setUp(self): + """Creates environment and instantiates placeholders for the test case.""" + + self.directory = tempfile.mkdtemp(dir=os.getcwd()) + spirv_args = self.test.spirv_args + # Instantiate placeholders in spirv_args + self.test.spirv_args = [ + arg.instantiate_for_spirv_args(self) + if isinstance(arg, PlaceHolder) else arg for arg in self.test.spirv_args + ] + # Get all shader files' names + self.inputs = [arg for arg in spirv_args if isinstance(arg, PlaceHolder)] + self.file_shaders = [arg.filename for arg in self.inputs] + + if 'environment' in get_all_variables(self.test): + self.test.environment.write(self.directory) + + expectations = [ + v for v in get_all_variables(self.test) + if v.startswith(EXPECTED_BEHAVIOR_PREFIX) + ] + # Instantiate placeholders in expectations + for expectation_name in expectations: + expectation = getattr(self.test, expectation_name) + if isinstance(expectation, list): + expanded_expections = [ + element.instantiate_for_expectation(self) + if isinstance(element, PlaceHolder) else element + for element in expectation + ] + setattr(self.test, expectation_name, expanded_expections) + elif isinstance(expectation, PlaceHolder): + setattr(self.test, expectation_name, + expectation.instantiate_for_expectation(self)) + + def tearDown(self): + """Removes the directory if we were not instructed to do otherwise.""" + if not self.test_manager.leave_output: + shutil.rmtree(self.directory) + + def runTest(self): + """Sets up and runs a test, reports any failures and then cleans up.""" + self.setUp() + success = False + message = '' + try: + self.command = [self.test_manager.executable_path] + self.command.extend(self.test.spirv_args) + + process = subprocess.Popen( + args=self.command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self.directory) + output = process.communicate(self.stdin_shader) + test_status = TestStatus(self.test_manager, process.returncode, output[0], + output[1], self.directory, self.inputs, + self.file_shaders) + run_results = [ + getattr(self.test, test_method)(test_status) + for test_method in get_all_test_methods(self.test.__class__) + ] + success, message = zip(*run_results) + success = all(success) + message = '\n'.join(message) + except Exception as e: + success = False + message = str(e) + self.test_manager.notify_result( + self, success, + message + '\nSTDOUT:\n%s\nSTDERR:\n%s' % (output[0], output[1])) + self.tearDown() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument( + 'spirv_tool', + metavar='path/to/spirv_tool', + type=str, + nargs=1, + help='Path to the spirv-* tool under test') + parser.add_argument( + 'spirv_as', + metavar='path/to/spirv-as', + type=str, + nargs=1, + help='Path to spirv-as') + parser.add_argument( + 'spirv_dis', + metavar='path/to/spirv-dis', + type=str, + nargs=1, + help='Path to spirv-dis') + parser.add_argument( + '--leave-output', + action='store_const', + const=1, + help='Do not clean up temporary directories') + parser.add_argument( + '--test-dir', nargs=1, help='Directory to gather the tests from') + args = parser.parse_args() + default_path = sys.path + root_dir = os.getcwd() + if args.test_dir: + root_dir = args.test_dir[0] + manager = TestManager(args.spirv_tool[0], args.spirv_as[0], args.spirv_dis[0]) + if args.leave_output: + manager.leave_output = True + for root, _, filenames in os.walk(root_dir): + for filename in fnmatch.filter(filenames, '*.py'): + if filename.endswith('nosetest.py'): + # Skip nose tests, which are for testing functions of + # the test framework. + continue + sys.path = default_path + sys.path.append(root) + mod = __import__(os.path.splitext(filename)[0]) + for _, obj, in inspect.getmembers(mod): + if inspect.isclass(obj) and hasattr(obj, 'parent_testsuite'): + manager.add_test(obj.parent_testsuite, obj()) + manager.run_tests() + if manager.num_failures > 0: + sys.exit(-1) + + +if __name__ == '__main__': + main() diff --git a/test/tools/spirv_test_framework_nosetest.py b/test/tools/spirv_test_framework_nosetest.py new file mode 100755 index 000000000..c0fbed581 --- /dev/null +++ b/test/tools/spirv_test_framework_nosetest.py @@ -0,0 +1,155 @@ +# Copyright (c) 2018 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from spirv_test_framework import get_all_test_methods, get_all_superclasses +from nose.tools import assert_equal, with_setup + + +# Classes to be used in testing get_all_{superclasses|test_methods}() +class Root: + + def check_root(self): + pass + + +class A(Root): + + def check_a(self): + pass + + +class B(Root): + + def check_b(self): + pass + + +class C(Root): + + def check_c(self): + pass + + +class D(Root): + + def check_d(self): + pass + + +class E(Root): + + def check_e(self): + pass + + +class H(B, C, D): + + def check_h(self): + pass + + +class I(E): + + def check_i(self): + pass + + +class O(H, I): + + def check_o(self): + pass + + +class U(A, O): + + def check_u(self): + pass + + +class X(U, A): + + def check_x(self): + pass + + +class R1: + + def check_r1(self): + pass + + +class R2: + + def check_r2(self): + pass + + +class Multi(R1, R2): + + def check_multi(self): + pass + + +def nosetest_get_all_superclasses(): + """Tests get_all_superclasses().""" + + assert_equal(get_all_superclasses(A), [Root]) + assert_equal(get_all_superclasses(B), [Root]) + assert_equal(get_all_superclasses(C), [Root]) + assert_equal(get_all_superclasses(D), [Root]) + assert_equal(get_all_superclasses(E), [Root]) + + assert_equal(get_all_superclasses(H), [Root, B, C, D]) + assert_equal(get_all_superclasses(I), [Root, E]) + + assert_equal(get_all_superclasses(O), [Root, B, C, D, E, H, I]) + + assert_equal(get_all_superclasses(U), [Root, B, C, D, E, H, I, A, O]) + assert_equal(get_all_superclasses(X), [Root, B, C, D, E, H, I, A, O, U]) + + assert_equal(get_all_superclasses(Multi), [R1, R2]) + + +def nosetest_get_all_methods(): + """Tests get_all_test_methods().""" + assert_equal(get_all_test_methods(A), ['check_root', 'check_a']) + assert_equal(get_all_test_methods(B), ['check_root', 'check_b']) + assert_equal(get_all_test_methods(C), ['check_root', 'check_c']) + assert_equal(get_all_test_methods(D), ['check_root', 'check_d']) + assert_equal(get_all_test_methods(E), ['check_root', 'check_e']) + + assert_equal( + get_all_test_methods(H), + ['check_root', 'check_b', 'check_c', 'check_d', 'check_h']) + assert_equal(get_all_test_methods(I), ['check_root', 'check_e', 'check_i']) + + assert_equal( + get_all_test_methods(O), [ + 'check_root', 'check_b', 'check_c', 'check_d', 'check_e', 'check_h', + 'check_i', 'check_o' + ]) + + assert_equal( + get_all_test_methods(U), [ + 'check_root', 'check_b', 'check_c', 'check_d', 'check_e', 'check_h', + 'check_i', 'check_a', 'check_o', 'check_u' + ]) + assert_equal( + get_all_test_methods(X), [ + 'check_root', 'check_b', 'check_c', 'check_d', 'check_e', 'check_h', + 'check_i', 'check_a', 'check_o', 'check_u', 'check_x' + ]) + + assert_equal( + get_all_test_methods(Multi), ['check_r1', 'check_r2', 'check_multi']) diff --git a/tools/opt/opt.cpp b/tools/opt/opt.cpp index 144087124..fcd260e45 100644 --- a/tools/opt/opt.cpp +++ b/tools/opt/opt.cpp @@ -138,7 +138,7 @@ Options (in lexicographical order): --eliminate-dead-functions Deletes functions that cannot be reached from entry points or exported functions. - --eliminate-dead-insert + --eliminate-dead-inserts Deletes unreferenced inserts into composites, most notably unused stores to vector components, that are not removed by aggressive dead code elimination. @@ -383,10 +383,20 @@ bool ReadFlagsFromFile(const char* oconfig_flag, return false; } - while (!input_file.eof()) { - std::string flag; - input_file >> flag; - if (flag.length() > 0 && flag[0] != '#') { + std::string line; + while (std::getline(input_file, line)) { + // Ignore empty lines and lines starting with the comment marker '#'. + if (line.length() == 0 || line[0] == '#') { + continue; + } + + // Tokenize the line. Add all found tokens to the list of found flags. This + // mimics the way the shell will parse whitespace on the command line. NOTE: + // This does not support quoting and it is not intended to. + std::istringstream iss(line); + while (!iss.eof()) { + std::string flag; + iss >> flag; file_flags->push_back(flag); } }