Merge pull request #7 from KhronosGroup/out-of-tree-testing

Improve out of tree testing, and avoid separate GL/Vulkan folders

CMakeLists.txt

@@ -55,11 +55,7 @@ find_program(PYTHON3_EXE python3)
 if(${PYTHON3_EXE} MATCHES "NOTFOUND")
   message(WARNING "Testing disabled. Could not find python3")
 else()
-  add_test(NAME spirv-cross-opengl-test
+  add_test(NAME spirv-cross-test
 	   COMMAND ${PYTHON3_EXE} ${CMAKE_CURRENT_SOURCE_DIR}/test_shaders.py
 		   ${CMAKE_CURRENT_SOURCE_DIR}/shaders)
-  add_test(NAME spirv-cross-vulkan-test
-	   COMMAND ${PYTHON3_EXE} ${CMAKE_CURRENT_SOURCE_DIR}/test_shaders.py
-		   --vulkan
-		   ${CMAKE_CURRENT_SOURCE_DIR}/shaders-vulkan)
 endif()

reference/shaders/vulkan/frag/push-constant.vk.frag.vk

@@ -0,0 +1,18 @@
+#version 310 es
+precision mediump float;
+precision highp int;
+
+layout(push_constant, std430) uniform PushConstants
+{
+    vec4 value0;
+    vec4 value1;
+} push;
+
+layout(location = 0) out vec4 FragColor;
+layout(location = 0) in vec4 vColor;
+
+void main()
+{
+    FragColor = ((vColor + push.value0) + push.value1);
+}
+

reference/shaders/vulkan/vert/vulkan-vertex.vk.vert

@@ -0,0 +1,9 @@
+#version 310 es
+
+uniform int SPIRV_Cross_BaseInstance;
+
+void main()
+{
+    gl_Position = (vec4(1.0, 2.0, 3.0, 4.0) * float((gl_VertexID + (gl_InstanceID + SPIRV_Cross_BaseInstance))));
+}
+

reference/shaders/vulkan/vert/vulkan-vertex.vk.vert.vk

@@ -0,0 +1,7 @@
+#version 310 es
+
+void main()
+{
+    gl_Position = (vec4(1.0, 2.0, 3.0, 4.0) * float((gl_VertexIndex + gl_InstanceIndex)));
+}
+

test_shaders.py

@@ -78,13 +78,15 @@ def cross_compile(shader, vulkan):
 
     subprocess.check_call(['glslangValidator', '-V' if vulkan else '-G', '-o', spirv_path, shader])
 
-    subprocess.check_call(['./spirv-cross', '--output', glsl_path, spirv_path])
+    spirv_cross_path = './spirv-cross'
+    subprocess.check_call([spirv_cross_path, '--output', glsl_path, spirv_path])
 
+    # A shader might not be possible to make valid GLSL from, skip validation for this case.
     if not ('nocompat' in glsl_path):
         validate_shader(glsl_path, False)
 
     if vulkan:
-        subprocess.check_call(['./spirv-cross', '--vulkan-semantics', '--output', vulkan_glsl_path, spirv_path])
+        subprocess.check_call([spirv_cross_path, '--vulkan-semantics', '--output', vulkan_glsl_path, spirv_path])
         validate_shader(vulkan_glsl_path, vulkan)
 
     return (spirv_path, glsl_path, vulkan_glsl_path if vulkan else None)
@@ -101,8 +103,17 @@ def make_reference_dir(path):
     if not os.path.exists(base):
         os.makedirs(base)
 
+def reference_path(directory, relpath):
+    split_paths = os.path.split(directory)
+    reference_dir = os.path.join(split_paths[0], 'reference/')
+    reference_dir = os.path.join(reference_dir, split_paths[1])
+    return os.path.join(reference_dir, relpath)
+
 def regression_check(shader, glsl, update, keep):
-    reference = os.path.join('./reference', shader)
+    reference = reference_path(shader[0], shader[1])
+    joined_path = os.path.join(shader[0], shader[1])
+    print('Reference shader path:', reference)
+
     if os.path.exists(reference):
         if md5_for_file(glsl) != md5_for_file(reference):
             if update:
@@ -121,46 +132,54 @@ def regression_check(shader, glsl, update, keep):
         else:
             os.remove(glsl)
     else:
-        print('Found new shader {}. Placing GLSL in {}'.format(shader, reference))
+        print('Found new shader {}. Placing GLSL in {}'.format(joined_path, reference))
         make_reference_dir(reference)
         shutil.move(glsl, reference)
 
-def test_shader(stats, shader, update, keep, vulkan):
-    print('Testing shader:', shader)
-    spirv, glsl, vulkan_glsl = cross_compile(shader, vulkan)
+def shader_is_vulkan(shader):
+    return '.vk.' in shader
 
-    if stats:
+def test_shader(stats, shader, update, keep):
+    joined_path = os.path.join(shader[0], shader[1])
+    vulkan = shader_is_vulkan(shader[1])
+
+    print('Testing shader:', joined_path)
+    spirv, glsl, vulkan_glsl = cross_compile(joined_path, vulkan)
+
+    # Only test GLSL stats if we have a shader following GL semantics.
+    if stats and (not vulkan):
         cross_stats = get_shader_stats(glsl)
 
     regression_check(shader, glsl, update, keep)
     if vulkan_glsl:
-        regression_check(shader + '.vk', vulkan_glsl, update, keep)
+        regression_check((shader[0], shader[1] + '.vk'), vulkan_glsl, update, keep)
     os.remove(spirv)
 
-    if stats:
-        pristine_stats = get_shader_stats(shader)
+    if stats and (not vulkan):
+        pristine_stats = get_shader_stats(joined_path)
 
         a = []
-        a.append(shader)
+        a.append(shader[1])
         for i in pristine_stats:
             a.append(str(i))
         for i in cross_stats:
             a.append(str(i))
         print(','.join(a), file = stats)
 
-def test_shaders(shader_dir, update, malisc, keep, vulkan):
+def test_shaders_helper(stats, shader_dir, update, malisc, keep):
+    for root, dirs, files in os.walk(os.path.join(shader_dir)):
+        for i in files:
+            path = os.path.join(root, i)
+            relpath = os.path.relpath(path, shader_dir)
+            test_shader(stats, (shader_dir, relpath), update, keep)
+
+def test_shaders(shader_dir, update, malisc, keep):
     if malisc:
         with open('stats.csv', 'w') as stats:
             print('Shader,OrigRegs,OrigUniRegs,OrigALUShort,OrigLSShort,OrigTEXShort,OrigALULong,OrigLSLong,OrigTEXLong,CrossRegs,CrossUniRegs,CrossALUShort,CrossLSShort,CrossTEXShort,CrossALULong,CrossLSLong,CrossTEXLong', file = stats)
-            for f in os.walk(os.path.join(shader_dir)):
-                for i in f[2]:
-                    shader = os.path.join(f[0], i)
-                    test_shader(stats, shader, update, keep, vulkan)
+            test_shaders_helper(stats, shader_dir, update, malisc, keep)
     else:
-        for f in os.walk(os.path.join(shader_dir)):
-            for i in f[2]:
-                shader = os.path.join(f[0], i)
-                test_shader(None, shader, update, keep, vulkan)
+        test_shaders_helper(None, shader_dir, update, malisc, keep)
 
 def main():
     parser = argparse.ArgumentParser(description = 'Script for regression testing.')
@@ -175,16 +194,13 @@ def main():
     parser.add_argument('--malisc',
             action = 'store_true',
             help = 'Use malisc offline compiler to determine static cycle counts before and after spirv-cross.')
-    parser.add_argument('--vulkan',
-            action = 'store_true',
-            help = 'Test shaders using Vulkan semantics instead of OpenGL.')
     args = parser.parse_args()
 
     if not args.folder:
         sys.stderr.write('Need shader folder.\n')
         sys.exit(1)
 
-    test_shaders(args.folder, args.update, args.malisc, args.keep, args.vulkan)
+    test_shaders(args.folder, args.update, args.malisc, args.keep)
     if args.malisc:
         print('Stats in stats.csv!')
     print('Tests completed!')