2016-03-02 17:09:16 +00:00
|
|
|
#!/usr/bin/env python3
|
|
|
|
|
|
2021-01-14 15:07:49 +00:00
|
|
|
# Copyright 2015-2021 Arm Limited
|
2021-05-08 08:47:48 +00:00
|
|
|
# SPDX-License-Identifier: Apache-2.0
|
2020-01-16 14:24:37 +00:00
|
|
|
#
|
|
|
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
|
|
|
# you may not use this file except in compliance with the License.
|
|
|
|
|
# You may obtain a copy of the License at
|
|
|
|
|
#
|
|
|
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
|
#
|
|
|
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
|
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
|
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
|
|
|
# See the License for the specific language governing permissions and
|
|
|
|
|
# limitations under the License.
|
2019-11-26 19:46:19 +00:00
|
|
|
|
2016-03-02 17:09:16 +00:00
|
|
|
import sys
|
|
|
|
|
import os
|
2017-01-25 06:26:39 +00:00
|
|
|
import os.path
|
2016-03-02 17:09:16 +00:00
|
|
|
import subprocess
|
|
|
|
|
import tempfile
|
|
|
|
|
import re
|
|
|
|
|
import itertools
|
|
|
|
|
import hashlib
|
|
|
|
|
import shutil
|
2016-03-22 13:44:12 +00:00
|
|
|
import argparse
|
2017-02-04 09:19:44 +00:00
|
|
|
import codecs
|
2018-06-03 18:16:37 +00:00
|
|
|
import json
|
2018-06-19 21:35:25 +00:00
|
|
|
import multiprocessing
|
2018-07-02 11:12:58 +00:00
|
|
|
import errno
|
2023-11-28 14:53:27 +00:00
|
|
|
import platform
|
2018-06-19 21:35:25 +00:00
|
|
|
from functools import partial
|
2016-03-02 17:09:16 +00:00
|
|
|
|
2019-03-07 11:36:16 +00:00
|
|
|
class Paths():
|
2019-04-12 12:44:24 +00:00
|
|
|
def __init__(self, spirv_cross, glslang, spirv_as, spirv_val, spirv_opt):
|
|
|
|
|
self.spirv_cross = spirv_cross
|
2019-03-07 11:36:16 +00:00
|
|
|
self.glslang = glslang
|
|
|
|
|
self.spirv_as = spirv_as
|
|
|
|
|
self.spirv_val = spirv_val
|
|
|
|
|
self.spirv_opt = spirv_opt
|
|
|
|
|
|
2018-04-27 08:31:39 +00:00
|
|
|
def remove_file(path):
|
|
|
|
|
#print('Removing file:', path)
|
|
|
|
|
os.remove(path)
|
|
|
|
|
|
|
|
|
|
def create_temporary(suff = ''):
|
|
|
|
|
f, path = tempfile.mkstemp(suffix = suff)
|
|
|
|
|
os.close(f)
|
|
|
|
|
#print('Creating temporary:', path)
|
|
|
|
|
return path
|
|
|
|
|
|
2016-03-02 17:09:16 +00:00
|
|
|
def parse_stats(stats):
|
|
|
|
|
m = re.search('([0-9]+) work registers', stats)
|
|
|
|
|
registers = int(m.group(1)) if m else 0
|
|
|
|
|
|
|
|
|
|
m = re.search('([0-9]+) uniform registers', stats)
|
|
|
|
|
uniform_regs = int(m.group(1)) if m else 0
|
|
|
|
|
|
|
|
|
|
m_list = re.findall('(-?[0-9]+)\s+(-?[0-9]+)\s+(-?[0-9]+)', stats)
|
|
|
|
|
alu_short = float(m_list[1][0]) if m_list else 0
|
|
|
|
|
ls_short = float(m_list[1][1]) if m_list else 0
|
|
|
|
|
tex_short = float(m_list[1][2]) if m_list else 0
|
|
|
|
|
alu_long = float(m_list[2][0]) if m_list else 0
|
|
|
|
|
ls_long = float(m_list[2][1]) if m_list else 0
|
|
|
|
|
tex_long = float(m_list[2][2]) if m_list else 0
|
|
|
|
|
|
|
|
|
|
return (registers, uniform_regs, alu_short, ls_short, tex_short, alu_long, ls_long, tex_long)
|
|
|
|
|
|
|
|
|
|
def get_shader_type(shader):
|
|
|
|
|
_, ext = os.path.splitext(shader)
|
|
|
|
|
if ext == '.vert':
|
|
|
|
|
return '--vertex'
|
|
|
|
|
elif ext == '.frag':
|
|
|
|
|
return '--fragment'
|
|
|
|
|
elif ext == '.comp':
|
|
|
|
|
return '--compute'
|
|
|
|
|
elif ext == '.tesc':
|
|
|
|
|
return '--tessellation_control'
|
|
|
|
|
elif ext == '.tese':
|
|
|
|
|
return '--tessellation_evaluation'
|
|
|
|
|
elif ext == '.geom':
|
|
|
|
|
return '--geometry'
|
|
|
|
|
else:
|
|
|
|
|
return ''
|
|
|
|
|
|
|
|
|
|
def get_shader_stats(shader):
|
2018-04-27 08:31:39 +00:00
|
|
|
path = create_temporary()
|
2016-03-02 17:09:16 +00:00
|
|
|
|
|
|
|
|
p = subprocess.Popen(['malisc', get_shader_type(shader), '--core', 'Mali-T760', '-V', shader], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
|
|
|
|
|
stdout, stderr = p.communicate()
|
2018-04-27 08:31:39 +00:00
|
|
|
remove_file(path)
|
2016-03-02 17:09:16 +00:00
|
|
|
|
|
|
|
|
if p.returncode != 0:
|
|
|
|
|
print(stderr.decode('utf-8'))
|
|
|
|
|
raise OSError('malisc failed')
|
|
|
|
|
p.wait()
|
|
|
|
|
|
|
|
|
|
returned = stdout.decode('utf-8')
|
|
|
|
|
return parse_stats(returned)
|
|
|
|
|
|
2017-01-31 03:55:21 +00:00
|
|
|
def print_msl_compiler_version():
|
|
|
|
|
try:
|
2023-11-28 14:53:27 +00:00
|
|
|
if platform.system() == 'Darwin':
|
|
|
|
|
subprocess.check_call(['xcrun', '--sdk', 'iphoneos', 'metal', '--version'])
|
|
|
|
|
print('... are the Metal compiler characteristics.\n') # display after so xcrun FNF is silent
|
|
|
|
|
else:
|
|
|
|
|
# Use Metal Windows toolkit to test on Linux (Wine) and Windows.
|
|
|
|
|
print('Running on non-macOS system.')
|
|
|
|
|
subprocess.check_call(['metal', '-x', 'metal', '--version'])
|
|
|
|
|
|
2017-01-31 03:55:21 +00:00
|
|
|
except OSError as e:
|
2018-07-02 11:12:58 +00:00
|
|
|
if (e.errno != errno.ENOENT): # Ignore xcrun not found error
|
2017-01-31 03:55:21 +00:00
|
|
|
raise
|
2021-03-12 11:51:53 +00:00
|
|
|
print('Metal SDK is not present.\n')
|
2019-06-17 14:06:14 +00:00
|
|
|
except subprocess.CalledProcessError:
|
|
|
|
|
pass
|
|
|
|
|
|
2020-09-29 04:07:55 +00:00
|
|
|
def msl_compiler_supports_version(version):
|
2023-02-20 12:41:28 +00:00
|
|
|
try:
|
2023-11-28 14:53:27 +00:00
|
|
|
if platform.system() == 'Darwin':
|
|
|
|
|
subprocess.check_call(['xcrun', '--sdk', 'macosx', 'metal', '-x', 'metal', version, '-'],
|
|
|
|
|
stdin = subprocess.DEVNULL, stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL)
|
|
|
|
|
print('Current SDK supports MSL {0}. Enabling validation for MSL {0} shaders.'.format(version))
|
|
|
|
|
else:
|
|
|
|
|
print('Running on {}, assuming {} is supported.'.format(platform.system(), version))
|
|
|
|
|
# If we're running on non-macOS system, assume it's supported.
|
2023-02-20 12:41:28 +00:00
|
|
|
return True
|
|
|
|
|
except OSError as e:
|
|
|
|
|
print('Failed to check if MSL {} is not supported. It probably is not.'.format(version))
|
|
|
|
|
return False
|
|
|
|
|
except subprocess.CalledProcessError:
|
|
|
|
|
print('Current SDK does NOT support MSL {0}. Disabling validation for MSL {0} shaders.'.format(version))
|
|
|
|
|
return False
|
2017-01-31 03:55:21 +00:00
|
|
|
|
2018-09-07 07:33:34 +00:00
|
|
|
def path_to_msl_standard(shader):
|
2023-11-28 13:25:51 +00:00
|
|
|
if '.msl31.' in shader:
|
|
|
|
|
return '-std=metal3.1'
|
|
|
|
|
elif '.msl3.' in shader:
|
|
|
|
|
return '-std=metal3.0'
|
|
|
|
|
elif '.ios.' in shader:
|
|
|
|
|
if '.msl2.' in shader:
|
2018-09-27 01:06:05 +00:00
|
|
|
return '-std=ios-metal2.0'
|
|
|
|
|
elif '.msl21.' in shader:
|
|
|
|
|
return '-std=ios-metal2.1'
|
2019-06-12 07:30:41 +00:00
|
|
|
elif '.msl22.' in shader:
|
|
|
|
|
return '-std=ios-metal2.2'
|
2020-09-29 04:07:55 +00:00
|
|
|
elif '.msl23.' in shader:
|
|
|
|
|
return '-std=ios-metal2.3'
|
2021-10-21 09:46:45 +00:00
|
|
|
elif '.msl24.' in shader:
|
|
|
|
|
return '-std=ios-metal2.4'
|
2018-09-27 01:06:05 +00:00
|
|
|
elif '.msl11.' in shader:
|
|
|
|
|
return '-std=ios-metal1.1'
|
|
|
|
|
elif '.msl10.' in shader:
|
|
|
|
|
return '-std=ios-metal1.0'
|
|
|
|
|
else:
|
|
|
|
|
return '-std=ios-metal1.2'
|
2018-09-07 07:33:34 +00:00
|
|
|
else:
|
2023-11-28 13:25:51 +00:00
|
|
|
if '.msl2.' in shader:
|
2018-09-27 01:06:05 +00:00
|
|
|
return '-std=macos-metal2.0'
|
|
|
|
|
elif '.msl21.' in shader:
|
|
|
|
|
return '-std=macos-metal2.1'
|
2019-06-12 07:30:41 +00:00
|
|
|
elif '.msl22.' in shader:
|
|
|
|
|
return '-std=macos-metal2.2'
|
2020-09-29 04:07:55 +00:00
|
|
|
elif '.msl23.' in shader:
|
|
|
|
|
return '-std=macos-metal2.3'
|
2021-10-21 09:46:45 +00:00
|
|
|
elif '.msl24.' in shader:
|
|
|
|
|
return '-std=macos-metal2.4'
|
2018-09-27 01:06:05 +00:00
|
|
|
elif '.msl11.' in shader:
|
|
|
|
|
return '-std=macos-metal1.1'
|
|
|
|
|
else:
|
|
|
|
|
return '-std=macos-metal1.2'
|
2018-09-07 07:33:34 +00:00
|
|
|
|
2018-09-07 07:45:25 +00:00
|
|
|
def path_to_msl_standard_cli(shader):
|
2023-11-28 13:25:51 +00:00
|
|
|
if '.msl31.' in shader:
|
|
|
|
|
return '30100'
|
|
|
|
|
elif '.msl3.' in shader:
|
2023-02-15 08:04:44 +00:00
|
|
|
return '30000'
|
|
|
|
|
elif '.msl2.' in shader:
|
2018-09-07 07:45:25 +00:00
|
|
|
return '20000'
|
|
|
|
|
elif '.msl21.' in shader:
|
|
|
|
|
return '20100'
|
2019-06-12 07:30:41 +00:00
|
|
|
elif '.msl22.' in shader:
|
|
|
|
|
return '20200'
|
2020-09-29 04:07:55 +00:00
|
|
|
elif '.msl23.' in shader:
|
|
|
|
|
return '20300'
|
2021-10-21 09:46:45 +00:00
|
|
|
elif '.msl24.' in shader:
|
|
|
|
|
return '20400'
|
2018-09-07 07:45:25 +00:00
|
|
|
elif '.msl11.' in shader:
|
|
|
|
|
return '10100'
|
|
|
|
|
else:
|
|
|
|
|
return '10200'
|
|
|
|
|
|
2023-11-28 14:53:27 +00:00
|
|
|
ignore_win_metal_tool = False
|
2017-11-21 08:52:53 +00:00
|
|
|
def validate_shader_msl(shader, opt):
|
|
|
|
|
msl_path = reference_path(shader[0], shader[1], opt)
|
2023-11-28 14:53:27 +00:00
|
|
|
global ignore_win_metal_tool
|
2017-01-31 03:55:21 +00:00
|
|
|
try:
|
2018-09-27 01:06:05 +00:00
|
|
|
if '.ios.' in msl_path:
|
|
|
|
|
msl_os = 'iphoneos'
|
|
|
|
|
else:
|
|
|
|
|
msl_os = 'macosx'
|
2023-11-28 14:53:27 +00:00
|
|
|
|
|
|
|
|
if platform.system() == 'Darwin':
|
|
|
|
|
subprocess.check_call(['xcrun', '--sdk', msl_os, 'metal', '-x', 'metal', path_to_msl_standard(msl_path), '-Werror', '-Wno-unused-variable', msl_path])
|
|
|
|
|
print('Compiled Metal shader: ' + msl_path) # display after so xcrun FNF is silent
|
|
|
|
|
elif not ignore_win_metal_tool:
|
|
|
|
|
# Use Metal Windows toolkit to test on Linux (Wine) and Windows. Running offline tool on Linux gets weird.
|
|
|
|
|
# Normal winepath doesn't work, it must be Z:/abspath *exactly* for some bizarre reason.
|
|
|
|
|
target_path = msl_path if platform.system == 'Windows' else ('Z:' + os.path.abspath(msl_path))
|
|
|
|
|
subprocess.check_call(['metal', '-x', 'metal', path_to_msl_standard(msl_path), '-Werror', '-Wno-unused-variable', target_path])
|
|
|
|
|
|
2017-01-31 03:55:21 +00:00
|
|
|
except OSError as oe:
|
2023-11-28 14:53:27 +00:00
|
|
|
if (oe.errno != errno.ENOENT): # Ignore xcrun or metal not found error
|
2017-01-31 03:55:21 +00:00
|
|
|
raise
|
2023-11-28 14:53:27 +00:00
|
|
|
print('metal toolkit does not exist, ignoring further attempts to use it.')
|
|
|
|
|
ignore_win_metal_tool = True
|
2017-01-31 03:55:21 +00:00
|
|
|
except subprocess.CalledProcessError:
|
|
|
|
|
print('Error compiling Metal shader: ' + msl_path)
|
2019-01-14 10:23:46 +00:00
|
|
|
raise RuntimeError('Failed to compile Metal shader')
|
2016-03-22 13:44:12 +00:00
|
|
|
|
2019-04-08 09:53:13 +00:00
|
|
|
def cross_compile_msl(shader, spirv, opt, iterations, paths):
|
2018-04-27 08:31:39 +00:00
|
|
|
spirv_path = create_temporary()
|
|
|
|
|
msl_path = create_temporary(os.path.basename(shader))
|
2017-09-29 09:07:11 +00:00
|
|
|
|
2022-03-04 10:19:33 +00:00
|
|
|
spirv_16 = '.spv16.' in shader
|
2021-10-13 13:52:04 +00:00
|
|
|
spirv_14 = '.spv14.' in shader
|
2022-03-04 10:19:33 +00:00
|
|
|
|
|
|
|
|
if spirv_16:
|
|
|
|
|
spirv_env = 'spv1.6'
|
|
|
|
|
glslang_env = 'spirv1.6'
|
|
|
|
|
elif spirv_14:
|
|
|
|
|
spirv_env = 'vulkan1.1spv1.4'
|
|
|
|
|
glslang_env = 'spirv1.4'
|
|
|
|
|
else:
|
|
|
|
|
spirv_env = 'vulkan1.1'
|
|
|
|
|
glslang_env = 'vulkan1.1'
|
2020-01-06 10:47:26 +00:00
|
|
|
|
2023-01-11 21:34:44 +00:00
|
|
|
spirv_cmd = [paths.spirv_as, '--preserve-numeric-ids', '--target-env', spirv_env, '-o', spirv_path, shader]
|
2019-02-19 16:00:49 +00:00
|
|
|
|
2017-09-29 09:07:11 +00:00
|
|
|
if spirv:
|
2019-02-19 16:00:49 +00:00
|
|
|
subprocess.check_call(spirv_cmd)
|
2017-09-29 09:07:11 +00:00
|
|
|
else:
|
2021-10-13 13:52:04 +00:00
|
|
|
subprocess.check_call([paths.glslang, '--amb' ,'--target-env', glslang_env, '-V', '-o', spirv_path, shader])
|
2017-09-29 09:07:11 +00:00
|
|
|
|
2019-11-05 15:59:41 +00:00
|
|
|
if opt and (not shader_is_invalid_spirv(shader)):
|
2019-10-21 20:39:53 +00:00
|
|
|
if '.graphics-robust-access.' in shader:
|
|
|
|
|
subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '--graphics-robust-access', '-o', spirv_path, spirv_path])
|
|
|
|
|
else:
|
|
|
|
|
subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '-o', spirv_path, spirv_path])
|
2017-11-21 08:52:53 +00:00
|
|
|
|
2019-04-12 12:44:24 +00:00
|
|
|
spirv_cross_path = paths.spirv_cross
|
2018-04-03 12:26:24 +00:00
|
|
|
|
2021-01-04 08:40:11 +00:00
|
|
|
msl_args = [spirv_cross_path, '--output', msl_path, spirv_path, '--msl', '--iterations', str(iterations)]
|
2018-09-07 07:45:25 +00:00
|
|
|
msl_args.append('--msl-version')
|
|
|
|
|
msl_args.append(path_to_msl_standard_cli(shader))
|
2021-01-04 08:40:11 +00:00
|
|
|
if not '.nomain.' in shader:
|
|
|
|
|
msl_args.append('--entry')
|
|
|
|
|
msl_args.append('main')
|
2018-09-20 01:36:33 +00:00
|
|
|
if '.swizzle.' in shader:
|
|
|
|
|
msl_args.append('--msl-swizzle-texture-samples')
|
2018-09-27 01:06:05 +00:00
|
|
|
if '.ios.' in shader:
|
|
|
|
|
msl_args.append('--msl-ios')
|
2019-01-14 13:53:47 +00:00
|
|
|
if '.pad-fragment.' in shader:
|
|
|
|
|
msl_args.append('--msl-pad-fragment-output')
|
2019-01-08 22:33:32 +00:00
|
|
|
if '.capture.' in shader:
|
|
|
|
|
msl_args.append('--msl-capture-output')
|
2019-02-06 05:47:50 +00:00
|
|
|
if '.domain.' in shader:
|
|
|
|
|
msl_args.append('--msl-domain-lower-left')
|
2019-03-14 09:29:34 +00:00
|
|
|
if '.argument.' in shader:
|
|
|
|
|
msl_args.append('--msl-argument-buffers')
|
2023-07-16 19:41:52 +00:00
|
|
|
if '.argument-tier-1.' in shader:
|
|
|
|
|
msl_args.append('--msl-argument-buffer-tier')
|
|
|
|
|
msl_args.append('1')
|
2019-04-23 10:17:21 +00:00
|
|
|
if '.texture-buffer-native.' in shader:
|
|
|
|
|
msl_args.append('--msl-texture-buffer-native')
|
2019-09-27 19:49:54 +00:00
|
|
|
if '.framebuffer-fetch.' in shader:
|
|
|
|
|
msl_args.append('--msl-framebuffer-fetch')
|
2019-10-09 15:22:25 +00:00
|
|
|
if '.invariant-float-math.' in shader:
|
|
|
|
|
msl_args.append('--msl-invariant-float-math')
|
2019-09-27 19:49:54 +00:00
|
|
|
if '.emulate-cube-array.' in shader:
|
|
|
|
|
msl_args.append('--msl-emulate-cube-array')
|
2019-03-15 20:53:21 +00:00
|
|
|
if '.discrete.' in shader:
|
2019-03-15 13:07:03 +00:00
|
|
|
# Arbitrary for testing purposes.
|
2019-03-15 20:53:21 +00:00
|
|
|
msl_args.append('--msl-discrete-descriptor-set')
|
2019-03-15 13:07:03 +00:00
|
|
|
msl_args.append('2')
|
2019-03-15 20:53:21 +00:00
|
|
|
msl_args.append('--msl-discrete-descriptor-set')
|
2019-03-15 13:07:03 +00:00
|
|
|
msl_args.append('3')
|
2020-01-16 10:07:30 +00:00
|
|
|
if '.force-active.' in shader:
|
|
|
|
|
msl_args.append('--msl-force-active-argument-buffer-resources')
|
2019-05-28 11:41:46 +00:00
|
|
|
if '.line.' in shader:
|
|
|
|
|
msl_args.append('--emit-line-directives')
|
2019-05-31 17:06:20 +00:00
|
|
|
if '.multiview.' in shader:
|
|
|
|
|
msl_args.append('--msl-multiview')
|
2020-08-23 21:44:41 +00:00
|
|
|
if '.no-layered.' in shader:
|
|
|
|
|
msl_args.append('--msl-multiview-no-layered-rendering')
|
2019-07-13 02:50:50 +00:00
|
|
|
if '.viewfromdev.' in shader:
|
|
|
|
|
msl_args.append('--msl-view-index-from-device-index')
|
2019-07-22 18:08:04 +00:00
|
|
|
if '.dispatchbase.' in shader:
|
|
|
|
|
msl_args.append('--msl-dispatch-base')
|
MSL: Support dynamic offsets for buffers in argument buffers.
Vulkan has two types of buffer descriptors,
`VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC` and
`VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC`, which allow the client to
offset the buffers by an amount given when the descriptor set is bound
to a pipeline. Metal provides no direct support for this when the buffer
in question is in an argument buffer, so once again we're on our own.
These offsets cannot be stored or associated in any way with the
argument buffer itself, because they are set at bind time. Different
pipelines may have different offsets set. Therefore, we must use a
separate buffer, not in any argument buffer, to hold these offsets. Then
the shader must manually offset the buffer pointer.
This change fully supports arrays, including arrays of arrays, even
though Vulkan forbids them. It does not, however, support runtime
arrays. Perhaps later.
2019-09-06 04:14:12 +00:00
|
|
|
if '.dynamic-buffer.' in shader:
|
|
|
|
|
# Arbitrary for testing purposes.
|
|
|
|
|
msl_args.append('--msl-dynamic-buffer')
|
|
|
|
|
msl_args.append('0')
|
|
|
|
|
msl_args.append('0')
|
|
|
|
|
msl_args.append('--msl-dynamic-buffer')
|
|
|
|
|
msl_args.append('1')
|
|
|
|
|
msl_args.append('2')
|
2019-12-17 04:58:16 +00:00
|
|
|
if '.inline-block.' in shader:
|
|
|
|
|
# Arbitrary for testing purposes.
|
|
|
|
|
msl_args.append('--msl-inline-uniform-block')
|
|
|
|
|
msl_args.append('0')
|
|
|
|
|
msl_args.append('0')
|
2019-10-14 10:51:48 +00:00
|
|
|
if '.device-argument-buffer.' in shader:
|
|
|
|
|
msl_args.append('--msl-device-argument-buffer')
|
|
|
|
|
msl_args.append('0')
|
|
|
|
|
msl_args.append('--msl-device-argument-buffer')
|
|
|
|
|
msl_args.append('1')
|
2020-02-24 11:47:14 +00:00
|
|
|
if '.force-native-array.' in shader:
|
|
|
|
|
msl_args.append('--msl-force-native-arrays')
|
2020-03-26 10:21:23 +00:00
|
|
|
if '.zero-initialize.' in shader:
|
|
|
|
|
msl_args.append('--force-zero-initialized-variables')
|
2020-04-10 06:13:33 +00:00
|
|
|
if '.frag-output.' in shader:
|
|
|
|
|
# Arbitrary for testing purposes.
|
|
|
|
|
msl_args.append('--msl-disable-frag-depth-builtin')
|
|
|
|
|
msl_args.append('--msl-disable-frag-stencil-ref-builtin')
|
|
|
|
|
msl_args.append('--msl-enable-frag-output-mask')
|
|
|
|
|
msl_args.append('0x000000ca')
|
2020-04-20 07:48:20 +00:00
|
|
|
if '.no-user-varying.' in shader:
|
|
|
|
|
msl_args.append('--msl-no-clip-distance-user-varying')
|
2020-06-14 04:03:30 +00:00
|
|
|
if '.shader-inputs.' in shader:
|
|
|
|
|
# Arbitrary for testing purposes.
|
|
|
|
|
msl_args.append('--msl-shader-input')
|
|
|
|
|
msl_args.append('0')
|
|
|
|
|
msl_args.append('u8')
|
|
|
|
|
msl_args.append('2')
|
|
|
|
|
msl_args.append('--msl-shader-input')
|
|
|
|
|
msl_args.append('1')
|
|
|
|
|
msl_args.append('u16')
|
|
|
|
|
msl_args.append('3')
|
|
|
|
|
msl_args.append('--msl-shader-input')
|
|
|
|
|
msl_args.append('6')
|
|
|
|
|
msl_args.append('other')
|
|
|
|
|
msl_args.append('4')
|
MSL: Add support for processing more than one patch per workgroup.
This should hopefully reduce underutilization of the GPU, especially on
GPUs where the thread execution width is greater than the number of
control points.
This also simplifies initialization by reading the buffer directly
instead of using Metal's vertex-attribute-in-compute support. It turns
out the only way in which shader stages are allowed to differ in their
interfaces is in the number of components per vector; the base type must
be the same. Since we are using the raw buffer instead of attributes, we
can now also emit arrays and matrices directly into the buffer, instead
of flattening them and then unpacking them. Structs are still flattened,
however; this is due to the need to handle vectors with fewer components
than were output, and I think handling this while also directly emitting
structs could get ugly.
Another advantage of this scheme is that the extra invocations needed to
read the attributes when there were more input than output points are
now no more. The number of threads per workgroup is now lcm(SIMD-size,
output control points). This should ensure we always process a whole
number of patches per workgroup.
To avoid complexity handling indices in the tessellation control shader,
I've also changed the way vertex shaders for tessellation are handled.
They are now compute kernels using Metal's support for vertex-style
stage input. This lets us always emit vertices into the buffer in order
of vertex shader execution. Now we no longer have to deal with indexing
in the tessellation control shader. This also fixes a long-standing
issue where if an index were greater than the number of vertices to
draw, the vertex shader would wind up writing outside the buffer, and
the vertex would be lost.
This is a breaking change, and I know SPIRV-Cross has other clients, so
I've hidden this behind an option for now. In the future, I want to
remove this option and make it the default.
2020-02-21 03:38:28 +00:00
|
|
|
if '.multi-patch.' in shader:
|
|
|
|
|
msl_args.append('--msl-multi-patch-workgroup')
|
|
|
|
|
# Arbitrary for testing purposes.
|
|
|
|
|
msl_args.append('--msl-shader-input')
|
|
|
|
|
msl_args.append('0')
|
|
|
|
|
msl_args.append('any32')
|
|
|
|
|
msl_args.append('3')
|
|
|
|
|
msl_args.append('--msl-shader-input')
|
|
|
|
|
msl_args.append('1')
|
|
|
|
|
msl_args.append('any16')
|
|
|
|
|
msl_args.append('2')
|
2022-10-03 06:33:50 +00:00
|
|
|
if '.raw-tess-in.' in shader:
|
|
|
|
|
msl_args.append('--msl-raw-buffer-tese-input')
|
MSL: Add support for processing more than one patch per workgroup.
This should hopefully reduce underutilization of the GPU, especially on
GPUs where the thread execution width is greater than the number of
control points.
This also simplifies initialization by reading the buffer directly
instead of using Metal's vertex-attribute-in-compute support. It turns
out the only way in which shader stages are allowed to differ in their
interfaces is in the number of components per vector; the base type must
be the same. Since we are using the raw buffer instead of attributes, we
can now also emit arrays and matrices directly into the buffer, instead
of flattening them and then unpacking them. Structs are still flattened,
however; this is due to the need to handle vectors with fewer components
than were output, and I think handling this while also directly emitting
structs could get ugly.
Another advantage of this scheme is that the extra invocations needed to
read the attributes when there were more input than output points are
now no more. The number of threads per workgroup is now lcm(SIMD-size,
output control points). This should ensure we always process a whole
number of patches per workgroup.
To avoid complexity handling indices in the tessellation control shader,
I've also changed the way vertex shaders for tessellation are handled.
They are now compute kernels using Metal's support for vertex-style
stage input. This lets us always emit vertices into the buffer in order
of vertex shader execution. Now we no longer have to deal with indexing
in the tessellation control shader. This also fixes a long-standing
issue where if an index were greater than the number of vertices to
draw, the vertex shader would wind up writing outside the buffer, and
the vertex would be lost.
This is a breaking change, and I know SPIRV-Cross has other clients, so
I've hidden this behind an option for now. In the future, I want to
remove this option and make it the default.
2020-02-21 03:38:28 +00:00
|
|
|
if '.for-tess.' in shader:
|
|
|
|
|
msl_args.append('--msl-vertex-for-tessellation')
|
2020-07-22 16:37:17 +00:00
|
|
|
if '.fixed-sample-mask.' in shader:
|
|
|
|
|
msl_args.append('--msl-additional-fixed-sample-mask')
|
|
|
|
|
msl_args.append('0x00000022')
|
2020-08-28 00:24:20 +00:00
|
|
|
if '.arrayed-subpass.' in shader:
|
|
|
|
|
msl_args.append('--msl-arrayed-subpass-input')
|
2020-10-15 01:48:52 +00:00
|
|
|
if '.1d-as-2d.' in shader:
|
|
|
|
|
msl_args.append('--msl-texture-1d-as-2d')
|
2020-11-19 05:16:46 +00:00
|
|
|
if '.simd.' in shader:
|
|
|
|
|
msl_args.append('--msl-ios-use-simdgroup-functions')
|
|
|
|
|
if '.emulate-subgroup.' in shader:
|
|
|
|
|
msl_args.append('--msl-emulate-subgroups')
|
|
|
|
|
if '.fixed-subgroup.' in shader:
|
|
|
|
|
# Arbitrary for testing purposes.
|
|
|
|
|
msl_args.append('--msl-fixed-subgroup-size')
|
|
|
|
|
msl_args.append('32')
|
MSL: Adjust FragCoord for sample-rate shading.
In Metal, the `[[position]]` input to a fragment shader remains at
fragment center, even at sample rate, like OpenGL and Direct3D. In
Vulkan, however, when the fragment shader runs at sample rate, the
`FragCoord` builtin moves to the sample position in the framebuffer,
instead of the fragment center. To account for this difference, adjust
the `FragCoord`, if present, by the sample position. The -0.5 offset is
because the fragment center is at (0.5, 0.5).
Also, add an option to force sample-rate shading in a fragment shader.
Since Metal has no explicit control for this, this is done by adding a
dummy `[[sample_id]]` which is otherwise unused, if none is already
present. This is intended to be used from e.g. MoltenVK when a
pipeline's `minSampleShading` value is nonzero.
Instead of checking if any `Input` variables have `Sample`
interpolation, I've elected to check that the `SampleRateShading`
capability is present. Since `SampleId`, `SamplePosition`, and the
`Sample` interpolation decoration require this cap, this should be
equivalent for any valid SPIR-V module. If this isn't acceptable, let me
know.
2020-11-20 21:41:46 +00:00
|
|
|
if '.force-sample.' in shader:
|
|
|
|
|
msl_args.append('--msl-force-sample-rate-shading')
|
2022-11-08 01:35:29 +00:00
|
|
|
if '.discard-checks.' in shader:
|
|
|
|
|
msl_args.append('--msl-check-discarded-frag-stores')
|
2024-05-10 16:10:22 +00:00
|
|
|
if '.force-frag-with-side-effects-execution.' in shader:
|
|
|
|
|
msl_args.append('--msl-force-frag-with-side-effects-execution')
|
2023-01-18 10:39:01 +00:00
|
|
|
if '.lod-as-grad.' in shader:
|
|
|
|
|
msl_args.append('--msl-sample-dref-lod-array-as-grad')
|
2023-11-26 07:17:49 +00:00
|
|
|
if '.agx-cube-grad.' in shader:
|
|
|
|
|
msl_args.append('--msl-agx-manual-cube-grad-fixup')
|
2021-01-07 13:30:35 +00:00
|
|
|
if '.decoration-binding.' in shader:
|
|
|
|
|
msl_args.append('--msl-decoration-binding')
|
2023-07-16 19:41:52 +00:00
|
|
|
if '.rich-descriptor.' in shader:
|
2023-08-17 10:01:26 +00:00
|
|
|
msl_args.append('--msl-runtime-array-rich-descriptor')
|
2023-10-14 01:57:01 +00:00
|
|
|
if '.replace-recursive-inputs.' in shader:
|
|
|
|
|
msl_args.append('--msl-replace-recursive-inputs')
|
2021-04-06 09:49:07 +00:00
|
|
|
if '.mask-location-0.' in shader:
|
|
|
|
|
msl_args.append('--mask-stage-output-location')
|
|
|
|
|
msl_args.append('0')
|
|
|
|
|
msl_args.append('0')
|
|
|
|
|
if '.mask-location-1.' in shader:
|
|
|
|
|
msl_args.append('--mask-stage-output-location')
|
|
|
|
|
msl_args.append('1')
|
|
|
|
|
msl_args.append('0')
|
|
|
|
|
if '.mask-position.' in shader:
|
|
|
|
|
msl_args.append('--mask-stage-output-builtin')
|
|
|
|
|
msl_args.append('Position')
|
|
|
|
|
if '.mask-point-size.' in shader:
|
|
|
|
|
msl_args.append('--mask-stage-output-builtin')
|
|
|
|
|
msl_args.append('PointSize')
|
|
|
|
|
if '.mask-clip-distance.' in shader:
|
|
|
|
|
msl_args.append('--mask-stage-output-builtin')
|
|
|
|
|
msl_args.append('ClipDistance')
|
2022-03-03 10:04:45 +00:00
|
|
|
if '.relax-nan.' in shader:
|
|
|
|
|
msl_args.append('--relax-nan-checks')
|
2018-04-03 12:26:24 +00:00
|
|
|
|
|
|
|
|
subprocess.check_call(msl_args)
|
2018-04-11 08:28:39 +00:00
|
|
|
|
|
|
|
|
if not shader_is_invalid_spirv(msl_path):
|
2022-01-05 14:53:29 +00:00
|
|
|
subprocess.check_call([paths.spirv_val, '--allow-localsizeid', '--scalar-block-layout', '--target-env', spirv_env, spirv_path])
|
2018-04-11 08:28:39 +00:00
|
|
|
|
2017-01-18 18:05:57 +00:00
|
|
|
return (spirv_path, msl_path)
|
|
|
|
|
|
2017-12-12 10:03:46 +00:00
|
|
|
def shader_model_hlsl(shader):
|
|
|
|
|
if '.vert' in shader:
|
2018-09-12 07:44:35 +00:00
|
|
|
if '.sm30.' in shader:
|
|
|
|
|
return '-Tvs_3_0'
|
|
|
|
|
else:
|
|
|
|
|
return '-Tvs_5_1'
|
2017-12-12 10:03:46 +00:00
|
|
|
elif '.frag' in shader:
|
2018-09-12 07:44:35 +00:00
|
|
|
if '.sm30.' in shader:
|
|
|
|
|
return '-Tps_3_0'
|
|
|
|
|
else:
|
|
|
|
|
return '-Tps_5_1'
|
2017-12-12 10:03:46 +00:00
|
|
|
elif '.comp' in shader:
|
|
|
|
|
return '-Tcs_5_1'
|
2022-10-10 22:16:45 +00:00
|
|
|
elif '.mesh' in shader:
|
|
|
|
|
return '-Tms_6_5'
|
|
|
|
|
elif '.task' in shader:
|
|
|
|
|
return '-Tas_6_5'
|
2017-12-12 10:03:46 +00:00
|
|
|
else:
|
|
|
|
|
return None
|
|
|
|
|
|
2017-12-12 12:23:56 +00:00
|
|
|
def shader_to_win_path(shader):
|
|
|
|
|
# It's (very) convenient to be able to run HLSL testing in wine on Unix-likes, so support that.
|
|
|
|
|
try:
|
|
|
|
|
with subprocess.Popen(['winepath', '-w', shader], stdout = subprocess.PIPE, stderr = subprocess.PIPE) as f:
|
|
|
|
|
stdout_data, stderr_data = f.communicate()
|
|
|
|
|
return stdout_data.decode('utf-8')
|
|
|
|
|
except OSError as oe:
|
2018-07-02 11:12:58 +00:00
|
|
|
if (oe.errno != errno.ENOENT): # Ignore not found errors
|
2017-12-12 12:23:56 +00:00
|
|
|
return shader
|
|
|
|
|
except subprocess.CalledProcessError:
|
|
|
|
|
raise
|
|
|
|
|
|
2017-12-12 12:33:13 +00:00
|
|
|
return shader
|
|
|
|
|
|
2018-04-18 14:43:28 +00:00
|
|
|
ignore_fxc = False
|
2019-03-07 11:36:16 +00:00
|
|
|
def validate_shader_hlsl(shader, force_no_external_validation, paths):
|
2019-07-17 09:24:31 +00:00
|
|
|
test_glslang = True
|
|
|
|
|
if '.nonuniformresource.' in shader:
|
|
|
|
|
test_glslang = False
|
|
|
|
|
if '.fxconly.' in shader:
|
|
|
|
|
test_glslang = False
|
2022-10-10 22:16:45 +00:00
|
|
|
if '.task' in shader or '.mesh' in shader:
|
|
|
|
|
test_glslang = False
|
2019-07-17 09:24:31 +00:00
|
|
|
|
2020-07-01 09:37:03 +00:00
|
|
|
hlsl_args = [paths.glslang, '--amb', '-e', 'main', '-D', '--target-env', 'vulkan1.1', '-V', shader]
|
|
|
|
|
if '.sm30.' in shader:
|
|
|
|
|
hlsl_args.append('--hlsl-dx9-compatible')
|
|
|
|
|
|
2019-07-17 09:24:31 +00:00
|
|
|
if test_glslang:
|
2020-07-01 09:37:03 +00:00
|
|
|
subprocess.check_call(hlsl_args)
|
|
|
|
|
|
2017-12-12 12:23:56 +00:00
|
|
|
is_no_fxc = '.nofxc.' in shader
|
2018-04-18 14:43:28 +00:00
|
|
|
global ignore_fxc
|
2018-10-26 08:53:11 +00:00
|
|
|
if (not ignore_fxc) and (not force_no_external_validation) and (not is_no_fxc):
|
2017-12-12 12:23:56 +00:00
|
|
|
try:
|
|
|
|
|
win_path = shader_to_win_path(shader)
|
2019-05-13 12:58:27 +00:00
|
|
|
args = ['fxc', '-nologo', shader_model_hlsl(shader), win_path]
|
|
|
|
|
if '.nonuniformresource.' in shader:
|
|
|
|
|
args.append('/enable_unbounded_descriptor_tables')
|
|
|
|
|
subprocess.check_call(args)
|
2017-12-12 12:23:56 +00:00
|
|
|
except OSError as oe:
|
2018-07-02 11:12:58 +00:00
|
|
|
if (oe.errno != errno.ENOENT): # Ignore not found errors
|
2019-01-14 10:23:46 +00:00
|
|
|
print('Failed to run FXC.')
|
|
|
|
|
ignore_fxc = True
|
2017-12-12 12:23:56 +00:00
|
|
|
raise
|
2018-04-18 14:43:28 +00:00
|
|
|
else:
|
2019-01-14 10:23:46 +00:00
|
|
|
print('Could not find FXC.')
|
2018-04-18 14:43:28 +00:00
|
|
|
ignore_fxc = True
|
2017-12-12 12:23:56 +00:00
|
|
|
except subprocess.CalledProcessError:
|
|
|
|
|
print('Failed compiling HLSL shader:', shader, 'with FXC.')
|
2019-01-14 10:23:46 +00:00
|
|
|
raise RuntimeError('Failed compiling HLSL shader')
|
2017-01-26 10:46:29 +00:00
|
|
|
|
2017-06-30 08:34:21 +00:00
|
|
|
def shader_to_sm(shader):
|
2020-06-04 09:35:21 +00:00
|
|
|
if '.sm62.' in shader:
|
|
|
|
|
return '62'
|
|
|
|
|
elif '.sm60.' in shader:
|
2018-04-11 13:02:02 +00:00
|
|
|
return '60'
|
|
|
|
|
elif '.sm51.' in shader:
|
2017-06-30 08:34:21 +00:00
|
|
|
return '51'
|
2018-09-11 18:57:56 +00:00
|
|
|
elif '.sm30.' in shader:
|
|
|
|
|
return '30'
|
2017-06-30 08:34:21 +00:00
|
|
|
else:
|
|
|
|
|
return '50'
|
|
|
|
|
|
2019-04-08 09:53:13 +00:00
|
|
|
def cross_compile_hlsl(shader, spirv, opt, force_no_external_validation, iterations, paths):
|
2018-04-27 08:31:39 +00:00
|
|
|
spirv_path = create_temporary()
|
|
|
|
|
hlsl_path = create_temporary(os.path.basename(shader))
|
2017-09-29 09:07:11 +00:00
|
|
|
|
2022-03-04 10:19:33 +00:00
|
|
|
spirv_16 = '.spv16.' in shader
|
2021-10-13 13:52:04 +00:00
|
|
|
spirv_14 = '.spv14.' in shader
|
2022-03-04 10:19:33 +00:00
|
|
|
|
|
|
|
|
if spirv_16:
|
|
|
|
|
spirv_env = 'spv1.6'
|
|
|
|
|
glslang_env = 'spirv1.6'
|
|
|
|
|
elif spirv_14:
|
|
|
|
|
spirv_env = 'vulkan1.1spv1.4'
|
|
|
|
|
glslang_env = 'spirv1.4'
|
|
|
|
|
else:
|
|
|
|
|
spirv_env = 'vulkan1.1'
|
|
|
|
|
glslang_env = 'vulkan1.1'
|
|
|
|
|
|
2023-01-11 21:34:44 +00:00
|
|
|
spirv_cmd = [paths.spirv_as, '--preserve-numeric-ids', '--target-env', spirv_env, '-o', spirv_path, shader]
|
2019-02-19 16:00:49 +00:00
|
|
|
|
2017-09-29 09:07:11 +00:00
|
|
|
if spirv:
|
2019-02-19 16:00:49 +00:00
|
|
|
subprocess.check_call(spirv_cmd)
|
2017-09-29 09:07:11 +00:00
|
|
|
else:
|
2021-10-13 13:52:04 +00:00
|
|
|
subprocess.check_call([paths.glslang, '--amb', '--target-env', glslang_env, '-V', '-o', spirv_path, shader])
|
2017-09-29 09:07:11 +00:00
|
|
|
|
2019-11-05 15:59:41 +00:00
|
|
|
if opt and (not shader_is_invalid_spirv(hlsl_path)):
|
2019-03-07 11:36:16 +00:00
|
|
|
subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '-o', spirv_path, spirv_path])
|
2017-11-21 08:52:53 +00:00
|
|
|
|
2019-04-12 12:44:24 +00:00
|
|
|
spirv_cross_path = paths.spirv_cross
|
2017-06-30 08:34:21 +00:00
|
|
|
|
|
|
|
|
sm = shader_to_sm(shader)
|
2019-05-28 11:41:46 +00:00
|
|
|
|
|
|
|
|
hlsl_args = [spirv_cross_path, '--entry', 'main', '--output', hlsl_path, spirv_path, '--hlsl-enable-compat', '--hlsl', '--shader-model', sm, '--iterations', str(iterations)]
|
|
|
|
|
if '.line.' in shader:
|
|
|
|
|
hlsl_args.append('--emit-line-directives')
|
2022-02-16 12:53:24 +00:00
|
|
|
if '.flatten.' in shader:
|
|
|
|
|
hlsl_args.append('--flatten-ubo')
|
2020-03-04 15:32:52 +00:00
|
|
|
if '.force-uav.' in shader:
|
|
|
|
|
hlsl_args.append('--hlsl-force-storage-buffer-as-uav')
|
2020-03-26 10:21:23 +00:00
|
|
|
if '.zero-initialize.' in shader:
|
|
|
|
|
hlsl_args.append('--force-zero-initialized-variables')
|
2020-04-03 09:21:41 +00:00
|
|
|
if '.nonwritable-uav-texture.' in shader:
|
|
|
|
|
hlsl_args.append('--hlsl-nonwritable-uav-texture-as-srv')
|
2020-06-04 09:35:21 +00:00
|
|
|
if '.native-16bit.' in shader:
|
|
|
|
|
hlsl_args.append('--hlsl-enable-16bit-types')
|
2020-11-03 10:18:32 +00:00
|
|
|
if '.flatten-matrix-vertex-input.' in shader:
|
|
|
|
|
hlsl_args.append('--hlsl-flatten-matrix-vertex-input-semantics')
|
2022-03-03 10:04:45 +00:00
|
|
|
if '.relax-nan.' in shader:
|
|
|
|
|
hlsl_args.append('--relax-nan-checks')
|
2023-05-19 09:29:45 +00:00
|
|
|
if '.structured.' in shader:
|
|
|
|
|
hlsl_args.append('--hlsl-preserve-structured-buffers')
|
2023-06-18 14:19:49 +00:00
|
|
|
if '.flip-vert-y.' in shader:
|
|
|
|
|
hlsl_args.append('--flip-vert-y')
|
2020-03-26 10:21:23 +00:00
|
|
|
|
2019-05-28 11:41:46 +00:00
|
|
|
subprocess.check_call(hlsl_args)
|
2018-04-11 08:28:39 +00:00
|
|
|
|
|
|
|
|
if not shader_is_invalid_spirv(hlsl_path):
|
2022-01-05 14:53:29 +00:00
|
|
|
subprocess.check_call([paths.spirv_val, '--allow-localsizeid', '--scalar-block-layout', '--target-env', spirv_env, spirv_path])
|
2017-01-26 08:45:17 +00:00
|
|
|
|
2019-03-07 11:36:16 +00:00
|
|
|
validate_shader_hlsl(hlsl_path, force_no_external_validation, paths)
|
2019-11-05 15:59:41 +00:00
|
|
|
|
2017-01-26 09:06:05 +00:00
|
|
|
return (spirv_path, hlsl_path)
|
2017-01-26 08:45:17 +00:00
|
|
|
|
2019-04-08 09:53:13 +00:00
|
|
|
def cross_compile_reflect(shader, spirv, opt, iterations, paths):
|
2018-06-03 18:16:37 +00:00
|
|
|
spirv_path = create_temporary()
|
|
|
|
|
reflect_path = create_temporary(os.path.basename(shader))
|
|
|
|
|
|
2023-01-11 21:34:44 +00:00
|
|
|
spirv_cmd = [paths.spirv_as, '--preserve-numeric-ids', '--target-env', 'vulkan1.1', '-o', spirv_path, shader]
|
2019-02-19 16:00:49 +00:00
|
|
|
|
2018-06-03 18:16:37 +00:00
|
|
|
if spirv:
|
2019-02-19 16:00:49 +00:00
|
|
|
subprocess.check_call(spirv_cmd)
|
2018-06-03 18:16:37 +00:00
|
|
|
else:
|
2019-09-18 19:56:51 +00:00
|
|
|
subprocess.check_call([paths.glslang, '--amb', '--target-env', 'vulkan1.1', '-V', '-o', spirv_path, shader])
|
2018-06-03 18:16:37 +00:00
|
|
|
|
2019-11-05 15:59:41 +00:00
|
|
|
if opt and (not shader_is_invalid_spirv(reflect_path)):
|
2019-03-07 11:36:16 +00:00
|
|
|
subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '-o', spirv_path, spirv_path])
|
2018-06-03 18:16:37 +00:00
|
|
|
|
2019-04-12 12:44:24 +00:00
|
|
|
spirv_cross_path = paths.spirv_cross
|
2018-06-03 18:16:37 +00:00
|
|
|
|
|
|
|
|
sm = shader_to_sm(shader)
|
2019-04-08 09:53:13 +00:00
|
|
|
subprocess.check_call([spirv_cross_path, '--entry', 'main', '--output', reflect_path, spirv_path, '--reflect', '--iterations', str(iterations)])
|
2018-06-03 18:16:37 +00:00
|
|
|
return (spirv_path, reflect_path)
|
|
|
|
|
|
2019-03-07 11:36:16 +00:00
|
|
|
def validate_shader(shader, vulkan, paths):
|
2017-01-25 06:26:39 +00:00
|
|
|
if vulkan:
|
2021-01-08 10:37:29 +00:00
|
|
|
spirv_14 = '.spv14.' in shader
|
|
|
|
|
glslang_env = 'spirv1.4' if spirv_14 else 'vulkan1.1'
|
|
|
|
|
subprocess.check_call([paths.glslang, '--amb', '--target-env', glslang_env, '-V', shader])
|
2017-01-25 06:26:39 +00:00
|
|
|
else:
|
2019-03-07 11:36:16 +00:00
|
|
|
subprocess.check_call([paths.glslang, shader])
|
2017-01-25 06:26:39 +00:00
|
|
|
|
2022-05-27 09:51:34 +00:00
|
|
|
def cross_compile(shader, vulkan, spirv, invalid_spirv, eliminate, is_legacy, force_es, flatten_ubo, sso, flatten_dim, opt, push_ubo, iterations, paths):
|
2018-04-27 08:31:39 +00:00
|
|
|
spirv_path = create_temporary()
|
|
|
|
|
glsl_path = create_temporary(os.path.basename(shader))
|
2016-03-02 17:09:16 +00:00
|
|
|
|
2022-01-06 13:16:28 +00:00
|
|
|
spirv_16 = '.spv16.' in shader
|
2021-01-08 10:37:29 +00:00
|
|
|
spirv_14 = '.spv14.' in shader
|
2022-01-06 13:16:28 +00:00
|
|
|
if spirv_16:
|
|
|
|
|
spirv_env = 'spv1.6'
|
2022-03-04 10:19:33 +00:00
|
|
|
glslang_env = 'spirv1.6'
|
2022-01-06 13:16:28 +00:00
|
|
|
elif spirv_14:
|
|
|
|
|
spirv_env = 'vulkan1.1spv1.4'
|
2022-03-04 10:19:33 +00:00
|
|
|
glslang_env = 'spirv1.4'
|
2022-01-06 13:16:28 +00:00
|
|
|
else:
|
|
|
|
|
spirv_env = 'vulkan1.1'
|
2022-03-04 10:19:33 +00:00
|
|
|
glslang_env = 'vulkan1.1'
|
2020-01-06 10:47:26 +00:00
|
|
|
|
2016-05-10 21:39:41 +00:00
|
|
|
if vulkan or spirv:
|
2018-04-27 08:31:39 +00:00
|
|
|
vulkan_glsl_path = create_temporary('vk' + os.path.basename(shader))
|
2016-05-05 08:16:22 +00:00
|
|
|
|
2023-01-11 21:34:44 +00:00
|
|
|
spirv_cmd = [paths.spirv_as, '--preserve-numeric-ids', '--target-env', spirv_env, '-o', spirv_path, shader]
|
2019-02-19 16:00:49 +00:00
|
|
|
|
2016-05-10 21:39:41 +00:00
|
|
|
if spirv:
|
2019-02-19 16:00:49 +00:00
|
|
|
subprocess.check_call(spirv_cmd)
|
2016-05-10 21:39:41 +00:00
|
|
|
else:
|
2023-01-11 14:18:19 +00:00
|
|
|
glslang_cmd = [paths.glslang, '--amb', '--target-env', glslang_env, '-V', '-o', spirv_path, shader]
|
|
|
|
|
if '.g.' in shader:
|
|
|
|
|
glslang_cmd.append('-g')
|
|
|
|
|
if '.gV.' in shader:
|
|
|
|
|
glslang_cmd.append('-gV')
|
|
|
|
|
subprocess.check_call(glslang_cmd)
|
2016-05-10 21:39:41 +00:00
|
|
|
|
2017-11-21 15:52:01 +00:00
|
|
|
if opt and (not invalid_spirv):
|
2019-03-07 11:36:16 +00:00
|
|
|
subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '-o', spirv_path, spirv_path])
|
2017-11-21 08:52:53 +00:00
|
|
|
|
2016-09-12 18:11:30 +00:00
|
|
|
if not invalid_spirv:
|
2022-01-05 14:53:29 +00:00
|
|
|
subprocess.check_call([paths.spirv_val, '--allow-localsizeid', '--scalar-block-layout', '--target-env', spirv_env, spirv_path])
|
2016-05-05 08:16:22 +00:00
|
|
|
|
2019-04-08 09:53:13 +00:00
|
|
|
extra_args = ['--iterations', str(iterations)]
|
2017-01-16 22:19:49 +00:00
|
|
|
if eliminate:
|
|
|
|
|
extra_args += ['--remove-unused-variables']
|
2017-01-13 15:41:27 +00:00
|
|
|
if is_legacy:
|
2017-01-16 22:19:49 +00:00
|
|
|
extra_args += ['--version', '100', '--es']
|
2022-05-27 09:51:34 +00:00
|
|
|
if force_es:
|
|
|
|
|
extra_args += ['--version', '310', '--es']
|
2017-01-16 22:19:49 +00:00
|
|
|
if flatten_ubo:
|
|
|
|
|
extra_args += ['--flatten-ubo']
|
2017-05-22 13:30:43 +00:00
|
|
|
if sso:
|
|
|
|
|
extra_args += ['--separate-shader-objects']
|
2017-05-22 15:40:00 +00:00
|
|
|
if flatten_dim:
|
|
|
|
|
extra_args += ['--flatten-multidimensional-arrays']
|
2019-03-19 09:58:37 +00:00
|
|
|
if push_ubo:
|
|
|
|
|
extra_args += ['--glsl-emit-push-constant-as-ubo']
|
2019-05-28 11:41:46 +00:00
|
|
|
if '.line.' in shader:
|
|
|
|
|
extra_args += ['--emit-line-directives']
|
2019-07-25 09:07:14 +00:00
|
|
|
if '.no-samplerless.' in shader:
|
|
|
|
|
extra_args += ['--vulkan-glsl-disable-ext-samplerless-texture-functions']
|
2020-03-04 15:41:00 +00:00
|
|
|
if '.no-qualifier-deduction.' in shader:
|
|
|
|
|
extra_args += ['--disable-storage-image-qualifier-deduction']
|
2020-03-19 13:20:37 +00:00
|
|
|
if '.framebuffer-fetch.' in shader:
|
|
|
|
|
extra_args += ['--glsl-remap-ext-framebuffer-fetch', '0', '0']
|
|
|
|
|
extra_args += ['--glsl-remap-ext-framebuffer-fetch', '1', '1']
|
|
|
|
|
extra_args += ['--glsl-remap-ext-framebuffer-fetch', '2', '2']
|
|
|
|
|
extra_args += ['--glsl-remap-ext-framebuffer-fetch', '3', '3']
|
2021-05-21 12:21:13 +00:00
|
|
|
if '.framebuffer-fetch-noncoherent.' in shader:
|
|
|
|
|
extra_args += ['--glsl-ext-framebuffer-fetch-noncoherent']
|
2020-03-26 10:21:23 +00:00
|
|
|
if '.zero-initialize.' in shader:
|
|
|
|
|
extra_args += ['--force-zero-initialized-variables']
|
2020-07-28 13:15:24 +00:00
|
|
|
if '.force-flattened-io.' in shader:
|
|
|
|
|
extra_args += ['--glsl-force-flattened-io-blocks']
|
2022-03-03 10:04:45 +00:00
|
|
|
if '.relax-nan.' in shader:
|
|
|
|
|
extra_args.append('--relax-nan-checks')
|
2017-01-13 15:41:27 +00:00
|
|
|
|
2019-04-12 12:44:24 +00:00
|
|
|
spirv_cross_path = paths.spirv_cross
|
2016-05-05 08:16:22 +00:00
|
|
|
|
2016-05-11 17:55:57 +00:00
|
|
|
# A shader might not be possible to make valid GLSL from, skip validation for this case.
|
2020-03-19 13:20:37 +00:00
|
|
|
if (not ('nocompat' in glsl_path)) or (not vulkan):
|
2018-04-11 08:28:39 +00:00
|
|
|
subprocess.check_call([spirv_cross_path, '--entry', 'main', '--output', glsl_path, spirv_path] + extra_args)
|
2020-03-19 13:20:37 +00:00
|
|
|
if not 'nocompat' in glsl_path:
|
|
|
|
|
validate_shader(glsl_path, False, paths)
|
2018-04-11 08:28:39 +00:00
|
|
|
else:
|
2018-04-27 08:31:39 +00:00
|
|
|
remove_file(glsl_path)
|
2018-04-11 08:28:39 +00:00
|
|
|
glsl_path = None
|
2016-05-05 08:16:22 +00:00
|
|
|
|
2020-06-29 10:50:31 +00:00
|
|
|
if (vulkan or spirv) and (not is_legacy):
|
2021-01-08 09:47:46 +00:00
|
|
|
subprocess.check_call([spirv_cross_path, '--entry', 'main', '-V', '--output', vulkan_glsl_path, spirv_path] + extra_args)
|
2019-03-07 11:36:16 +00:00
|
|
|
validate_shader(vulkan_glsl_path, True, paths)
|
2018-04-27 08:31:39 +00:00
|
|
|
# SPIR-V shaders might just want to validate Vulkan GLSL output, we don't always care about the output.
|
|
|
|
|
if not vulkan:
|
|
|
|
|
remove_file(vulkan_glsl_path)
|
2016-05-05 08:16:22 +00:00
|
|
|
|
|
|
|
|
return (spirv_path, glsl_path, vulkan_glsl_path if vulkan else None)
|
2016-03-02 17:09:16 +00:00
|
|
|
|
2017-02-04 09:19:44 +00:00
|
|
|
def make_unix_newline(buf):
|
|
|
|
|
decoded = codecs.decode(buf, 'utf-8')
|
|
|
|
|
decoded = decoded.replace('\r', '')
|
|
|
|
|
return codecs.encode(decoded, 'utf-8')
|
|
|
|
|
|
2016-03-02 17:09:16 +00:00
|
|
|
def md5_for_file(path):
|
|
|
|
|
md5 = hashlib.md5()
|
|
|
|
|
with open(path, 'rb') as f:
|
2017-02-04 09:19:44 +00:00
|
|
|
for chunk in iter(lambda: make_unix_newline(f.read(8192)), b''):
|
2016-03-02 17:09:16 +00:00
|
|
|
md5.update(chunk)
|
|
|
|
|
return md5.digest()
|
|
|
|
|
|
|
|
|
|
def make_reference_dir(path):
|
|
|
|
|
base = os.path.dirname(path)
|
|
|
|
|
if not os.path.exists(base):
|
|
|
|
|
os.makedirs(base)
|
|
|
|
|
|
2017-11-21 08:52:53 +00:00
|
|
|
def reference_path(directory, relpath, opt):
|
2016-05-11 17:39:38 +00:00
|
|
|
split_paths = os.path.split(directory)
|
2017-11-21 08:52:53 +00:00
|
|
|
reference_dir = os.path.join(split_paths[0], 'reference/' + ('opt/' if opt else ''))
|
2016-05-11 17:39:38 +00:00
|
|
|
reference_dir = os.path.join(reference_dir, split_paths[1])
|
|
|
|
|
return os.path.join(reference_dir, relpath)
|
|
|
|
|
|
2019-04-08 09:53:13 +00:00
|
|
|
def regression_check_reflect(shader, json_file, args):
|
|
|
|
|
reference = reference_path(shader[0], shader[1], args.opt) + '.json'
|
2018-06-03 18:16:37 +00:00
|
|
|
joined_path = os.path.join(shader[0], shader[1])
|
|
|
|
|
print('Reference shader reflection path:', reference)
|
|
|
|
|
if os.path.exists(reference):
|
2019-10-04 09:04:52 +00:00
|
|
|
actual = md5_for_file(json_file)
|
|
|
|
|
expected = md5_for_file(reference)
|
|
|
|
|
if actual != expected:
|
2019-04-10 09:33:52 +00:00
|
|
|
if args.update:
|
2018-06-03 18:16:37 +00:00
|
|
|
print('Generated reflection json has changed for {}!'.format(reference))
|
|
|
|
|
# If we expect changes, update the reference file.
|
|
|
|
|
if os.path.exists(reference):
|
|
|
|
|
remove_file(reference)
|
|
|
|
|
make_reference_dir(reference)
|
|
|
|
|
shutil.move(json_file, reference)
|
|
|
|
|
else:
|
|
|
|
|
print('Generated reflection json in {} does not match reference {}!'.format(json_file, reference))
|
2021-12-09 16:29:12 +00:00
|
|
|
if args.diff:
|
|
|
|
|
diff_path = generate_diff_file(reference, glsl)
|
|
|
|
|
with open(diff_path, 'r') as f:
|
|
|
|
|
print('')
|
|
|
|
|
print('Diff:')
|
|
|
|
|
print(f.read())
|
|
|
|
|
print('')
|
|
|
|
|
remove_file(diff_path)
|
|
|
|
|
else:
|
|
|
|
|
with open(json_file, 'r') as f:
|
|
|
|
|
print('')
|
|
|
|
|
print('Generated:')
|
|
|
|
|
print('======================')
|
|
|
|
|
print(f.read())
|
|
|
|
|
print('======================')
|
|
|
|
|
print('')
|
2018-06-03 18:16:37 +00:00
|
|
|
|
|
|
|
|
# Otherwise, fail the test. Keep the shader file around so we can inspect.
|
2019-04-08 09:53:13 +00:00
|
|
|
if not args.keep:
|
2018-06-03 18:16:37 +00:00
|
|
|
remove_file(json_file)
|
2019-01-14 10:23:46 +00:00
|
|
|
|
|
|
|
|
raise RuntimeError('Does not match reference')
|
2018-06-03 18:16:37 +00:00
|
|
|
else:
|
|
|
|
|
remove_file(json_file)
|
|
|
|
|
else:
|
|
|
|
|
print('Found new shader {}. Placing generated source code in {}'.format(joined_path, reference))
|
|
|
|
|
make_reference_dir(reference)
|
|
|
|
|
shutil.move(json_file, reference)
|
2019-11-05 15:59:41 +00:00
|
|
|
|
2021-12-09 16:29:12 +00:00
|
|
|
def generate_diff_file(origin, generated):
|
|
|
|
|
diff_destination = create_temporary()
|
|
|
|
|
with open(diff_destination, "w") as f:
|
|
|
|
|
try:
|
|
|
|
|
subprocess.check_call(["diff", origin, generated], stdout=f)
|
|
|
|
|
except subprocess.CalledProcessError as e:
|
|
|
|
|
# diff returns 1 when the files are different so we can safely
|
|
|
|
|
# ignore this case.
|
|
|
|
|
if e.returncode != 1:
|
|
|
|
|
raise e
|
|
|
|
|
|
|
|
|
|
return diff_destination
|
|
|
|
|
|
2019-04-08 09:53:13 +00:00
|
|
|
def regression_check(shader, glsl, args):
|
|
|
|
|
reference = reference_path(shader[0], shader[1], args.opt)
|
2016-05-11 17:39:38 +00:00
|
|
|
joined_path = os.path.join(shader[0], shader[1])
|
|
|
|
|
print('Reference shader path:', reference)
|
|
|
|
|
|
2016-03-02 17:09:16 +00:00
|
|
|
if os.path.exists(reference):
|
|
|
|
|
if md5_for_file(glsl) != md5_for_file(reference):
|
2019-04-10 09:33:52 +00:00
|
|
|
if args.update:
|
2017-08-11 18:54:58 +00:00
|
|
|
print('Generated source code has changed for {}!'.format(reference))
|
2016-03-22 13:44:12 +00:00
|
|
|
# If we expect changes, update the reference file.
|
|
|
|
|
if os.path.exists(reference):
|
2018-04-27 08:31:39 +00:00
|
|
|
remove_file(reference)
|
2016-03-22 13:44:12 +00:00
|
|
|
make_reference_dir(reference)
|
|
|
|
|
shutil.move(glsl, reference)
|
|
|
|
|
else:
|
2017-08-11 18:54:58 +00:00
|
|
|
print('Generated source code in {} does not match reference {}!'.format(glsl, reference))
|
2021-12-09 16:29:12 +00:00
|
|
|
if args.diff:
|
|
|
|
|
diff_path = generate_diff_file(reference, glsl)
|
|
|
|
|
with open(diff_path, 'r') as f:
|
|
|
|
|
print('')
|
|
|
|
|
print('Diff:')
|
|
|
|
|
print(f.read())
|
|
|
|
|
print('')
|
|
|
|
|
remove_file(diff_path)
|
|
|
|
|
else:
|
|
|
|
|
with open(glsl, 'r') as f:
|
|
|
|
|
print('')
|
|
|
|
|
print('Generated:')
|
|
|
|
|
print('======================')
|
|
|
|
|
print(f.read())
|
|
|
|
|
print('======================')
|
|
|
|
|
print('')
|
2016-12-16 12:48:30 +00:00
|
|
|
|
2016-03-22 13:56:50 +00:00
|
|
|
# Otherwise, fail the test. Keep the shader file around so we can inspect.
|
2019-04-08 09:53:13 +00:00
|
|
|
if not args.keep:
|
2018-04-27 08:31:39 +00:00
|
|
|
remove_file(glsl)
|
2019-01-14 10:23:46 +00:00
|
|
|
raise RuntimeError('Does not match reference')
|
2016-03-02 17:09:16 +00:00
|
|
|
else:
|
2018-04-27 08:31:39 +00:00
|
|
|
remove_file(glsl)
|
2016-03-02 17:09:16 +00:00
|
|
|
else:
|
2017-08-11 18:54:58 +00:00
|
|
|
print('Found new shader {}. Placing generated source code in {}'.format(joined_path, reference))
|
2016-03-02 17:09:16 +00:00
|
|
|
make_reference_dir(reference)
|
|
|
|
|
shutil.move(glsl, reference)
|
|
|
|
|
|
2016-05-11 17:55:57 +00:00
|
|
|
def shader_is_vulkan(shader):
|
|
|
|
|
return '.vk.' in shader
|
|
|
|
|
|
2016-07-11 10:47:46 +00:00
|
|
|
def shader_is_desktop(shader):
|
|
|
|
|
return '.desktop.' in shader
|
|
|
|
|
|
2016-08-26 10:58:50 +00:00
|
|
|
def shader_is_eliminate_dead_variables(shader):
|
|
|
|
|
return '.noeliminate.' not in shader
|
|
|
|
|
|
2016-05-10 21:39:41 +00:00
|
|
|
def shader_is_spirv(shader):
|
|
|
|
|
return '.asm.' in shader
|
|
|
|
|
|
2016-09-12 18:11:30 +00:00
|
|
|
def shader_is_invalid_spirv(shader):
|
|
|
|
|
return '.invalid.' in shader
|
|
|
|
|
|
2017-01-13 15:41:27 +00:00
|
|
|
def shader_is_legacy(shader):
|
|
|
|
|
return '.legacy.' in shader
|
|
|
|
|
|
2022-05-27 09:51:34 +00:00
|
|
|
def shader_is_force_es(shader):
|
|
|
|
|
return '.es.' in shader
|
|
|
|
|
|
2017-01-16 22:19:49 +00:00
|
|
|
def shader_is_flatten_ubo(shader):
|
|
|
|
|
return '.flatten.' in shader
|
|
|
|
|
|
2017-05-22 13:30:43 +00:00
|
|
|
def shader_is_sso(shader):
|
|
|
|
|
return '.sso.' in shader
|
|
|
|
|
|
2017-05-22 15:40:00 +00:00
|
|
|
def shader_is_flatten_dimensions(shader):
|
|
|
|
|
return '.flatten_dim.' in shader
|
|
|
|
|
|
2017-11-22 10:04:29 +00:00
|
|
|
def shader_is_noopt(shader):
|
|
|
|
|
return '.noopt.' in shader
|
|
|
|
|
|
2019-03-19 09:58:37 +00:00
|
|
|
def shader_is_push_ubo(shader):
|
|
|
|
|
return '.push-ubo.' in shader
|
|
|
|
|
|
2019-04-08 09:53:13 +00:00
|
|
|
def test_shader(stats, shader, args, paths):
|
2016-05-11 17:39:38 +00:00
|
|
|
joined_path = os.path.join(shader[0], shader[1])
|
2016-05-11 17:55:57 +00:00
|
|
|
vulkan = shader_is_vulkan(shader[1])
|
2016-07-11 10:47:46 +00:00
|
|
|
desktop = shader_is_desktop(shader[1])
|
2016-08-26 10:58:50 +00:00
|
|
|
eliminate = shader_is_eliminate_dead_variables(shader[1])
|
2016-09-10 10:52:23 +00:00
|
|
|
is_spirv = shader_is_spirv(shader[1])
|
2016-09-12 18:11:30 +00:00
|
|
|
invalid_spirv = shader_is_invalid_spirv(shader[1])
|
2017-01-13 15:41:27 +00:00
|
|
|
is_legacy = shader_is_legacy(shader[1])
|
2022-05-27 09:51:34 +00:00
|
|
|
force_es = shader_is_force_es(shader[1])
|
2017-01-16 22:19:49 +00:00
|
|
|
flatten_ubo = shader_is_flatten_ubo(shader[1])
|
2017-05-22 13:30:43 +00:00
|
|
|
sso = shader_is_sso(shader[1])
|
2017-05-22 15:40:00 +00:00
|
|
|
flatten_dim = shader_is_flatten_dimensions(shader[1])
|
2017-11-22 10:04:29 +00:00
|
|
|
noopt = shader_is_noopt(shader[1])
|
2019-03-19 09:58:37 +00:00
|
|
|
push_ubo = shader_is_push_ubo(shader[1])
|
2016-05-11 17:39:38 +00:00
|
|
|
|
|
|
|
|
print('Testing shader:', joined_path)
|
2022-05-27 09:51:34 +00:00
|
|
|
spirv, glsl, vulkan_glsl = cross_compile(joined_path, vulkan, is_spirv, invalid_spirv, eliminate, is_legacy, force_es, flatten_ubo, sso, flatten_dim, args.opt and (not noopt), push_ubo, args.iterations, paths)
|
2016-03-02 17:09:16 +00:00
|
|
|
|
2016-05-11 17:55:57 +00:00
|
|
|
# Only test GLSL stats if we have a shader following GL semantics.
|
2016-09-10 10:52:23 +00:00
|
|
|
if stats and (not vulkan) and (not is_spirv) and (not desktop):
|
2016-03-22 13:44:12 +00:00
|
|
|
cross_stats = get_shader_stats(glsl)
|
|
|
|
|
|
2018-04-11 08:28:39 +00:00
|
|
|
if glsl:
|
2019-04-08 09:53:13 +00:00
|
|
|
regression_check(shader, glsl, args)
|
2016-05-05 08:16:22 +00:00
|
|
|
if vulkan_glsl:
|
2019-04-08 09:53:13 +00:00
|
|
|
regression_check((shader[0], shader[1] + '.vk'), vulkan_glsl, args)
|
2018-04-27 08:31:39 +00:00
|
|
|
|
|
|
|
|
remove_file(spirv)
|
2016-03-02 17:09:16 +00:00
|
|
|
|
2016-09-10 10:52:23 +00:00
|
|
|
if stats and (not vulkan) and (not is_spirv) and (not desktop):
|
2016-05-11 17:39:38 +00:00
|
|
|
pristine_stats = get_shader_stats(joined_path)
|
2016-03-22 13:44:12 +00:00
|
|
|
|
|
|
|
|
a = []
|
2016-05-11 17:39:38 +00:00
|
|
|
a.append(shader[1])
|
2016-03-22 13:44:12 +00:00
|
|
|
for i in pristine_stats:
|
|
|
|
|
a.append(str(i))
|
|
|
|
|
for i in cross_stats:
|
|
|
|
|
a.append(str(i))
|
|
|
|
|
print(','.join(a), file = stats)
|
|
|
|
|
|
2019-04-08 09:53:13 +00:00
|
|
|
def test_shader_msl(stats, shader, args, paths):
|
2017-01-18 18:05:57 +00:00
|
|
|
joined_path = os.path.join(shader[0], shader[1])
|
2017-01-31 03:55:21 +00:00
|
|
|
print('\nTesting MSL shader:', joined_path)
|
2017-09-29 09:07:11 +00:00
|
|
|
is_spirv = shader_is_spirv(shader[1])
|
2017-11-22 10:04:29 +00:00
|
|
|
noopt = shader_is_noopt(shader[1])
|
2019-04-08 09:53:13 +00:00
|
|
|
spirv, msl = cross_compile_msl(joined_path, is_spirv, args.opt and (not noopt), args.iterations, paths)
|
|
|
|
|
regression_check(shader, msl, args)
|
2017-12-26 21:32:45 +00:00
|
|
|
|
|
|
|
|
# Uncomment the following line to print the temp SPIR-V file path.
|
|
|
|
|
# This temp SPIR-V file is not deleted until after the Metal validation step below.
|
|
|
|
|
# If Metal validation fails, the temp SPIR-V file can be copied out and
|
|
|
|
|
# used as input to an invocation of spirv-cross to debug from Xcode directly.
|
|
|
|
|
# To do so, build spriv-cross using `make DEBUG=1`, then run the spriv-cross
|
|
|
|
|
# executable from Xcode using args: `--msl --entry main --output msl_path spirv_path`.
|
|
|
|
|
# print('SPRIV shader: ' + spirv)
|
2017-02-05 10:04:45 +00:00
|
|
|
|
2023-11-28 14:53:27 +00:00
|
|
|
shader_is_msl22 = '.msl22.' in joined_path
|
|
|
|
|
shader_is_msl23 = '.msl23.' in joined_path
|
|
|
|
|
shader_is_msl24 = '.msl24.' in joined_path
|
|
|
|
|
shader_is_msl30 = '.msl3.' in joined_path
|
|
|
|
|
shader_is_msl31 = '.msl31.' in joined_path
|
|
|
|
|
skip_validation = (shader_is_msl22 and (not args.msl22)) or \
|
|
|
|
|
(shader_is_msl23 and (not args.msl23)) or \
|
|
|
|
|
(shader_is_msl24 and (not args.msl24)) or \
|
|
|
|
|
(shader_is_msl30 and (not args.msl30)) or \
|
|
|
|
|
(shader_is_msl31 and (not args.msl31))
|
|
|
|
|
|
|
|
|
|
if skip_validation:
|
|
|
|
|
print('Skipping validation for {} due to lack of toolchain support.'.format(joined_path))
|
|
|
|
|
|
2019-06-18 08:35:17 +00:00
|
|
|
if '.invalid.' in joined_path:
|
|
|
|
|
skip_validation = True
|
2019-06-17 14:06:14 +00:00
|
|
|
|
|
|
|
|
if (not args.force_no_external_validation) and (not skip_validation):
|
2019-04-08 09:53:13 +00:00
|
|
|
validate_shader_msl(shader, args.opt)
|
2017-01-18 18:05:57 +00:00
|
|
|
|
2018-04-27 08:31:39 +00:00
|
|
|
remove_file(spirv)
|
2017-12-26 21:32:45 +00:00
|
|
|
|
2019-04-08 09:53:13 +00:00
|
|
|
def test_shader_hlsl(stats, shader, args, paths):
|
2017-01-26 08:45:17 +00:00
|
|
|
joined_path = os.path.join(shader[0], shader[1])
|
|
|
|
|
print('Testing HLSL shader:', joined_path)
|
2017-09-29 09:07:11 +00:00
|
|
|
is_spirv = shader_is_spirv(shader[1])
|
2017-11-22 10:04:29 +00:00
|
|
|
noopt = shader_is_noopt(shader[1])
|
2019-04-08 09:53:13 +00:00
|
|
|
spirv, hlsl = cross_compile_hlsl(joined_path, is_spirv, args.opt and (not noopt), args.force_no_external_validation, args.iterations, paths)
|
|
|
|
|
regression_check(shader, hlsl, args)
|
2018-04-27 08:31:39 +00:00
|
|
|
remove_file(spirv)
|
2017-01-26 08:45:17 +00:00
|
|
|
|
2019-04-08 09:53:13 +00:00
|
|
|
def test_shader_reflect(stats, shader, args, paths):
|
2018-06-03 18:16:37 +00:00
|
|
|
joined_path = os.path.join(shader[0], shader[1])
|
|
|
|
|
print('Testing shader reflection:', joined_path)
|
|
|
|
|
is_spirv = shader_is_spirv(shader[1])
|
|
|
|
|
noopt = shader_is_noopt(shader[1])
|
2019-04-08 09:53:13 +00:00
|
|
|
spirv, reflect = cross_compile_reflect(joined_path, is_spirv, args.opt and (not noopt), args.iterations, paths)
|
|
|
|
|
regression_check_reflect(shader, reflect, args)
|
2018-06-03 18:16:37 +00:00
|
|
|
remove_file(spirv)
|
|
|
|
|
|
2019-03-07 11:43:00 +00:00
|
|
|
def test_shader_file(relpath, stats, args, backend):
|
2019-04-12 12:44:24 +00:00
|
|
|
paths = Paths(args.spirv_cross, args.glslang, args.spirv_as, args.spirv_val, args.spirv_opt)
|
2019-01-14 10:23:46 +00:00
|
|
|
try:
|
|
|
|
|
if backend == 'msl':
|
2019-04-08 09:53:13 +00:00
|
|
|
test_shader_msl(stats, (args.folder, relpath), args, paths)
|
2019-01-14 10:23:46 +00:00
|
|
|
elif backend == 'hlsl':
|
2019-04-08 09:53:13 +00:00
|
|
|
test_shader_hlsl(stats, (args.folder, relpath), args, paths)
|
2019-01-14 10:23:46 +00:00
|
|
|
elif backend == 'reflect':
|
2019-04-08 09:53:13 +00:00
|
|
|
test_shader_reflect(stats, (args.folder, relpath), args, paths)
|
2019-01-14 10:23:46 +00:00
|
|
|
else:
|
2019-04-08 09:53:13 +00:00
|
|
|
test_shader(stats, (args.folder, relpath), args, paths)
|
2019-01-14 10:23:46 +00:00
|
|
|
return None
|
|
|
|
|
except Exception as e:
|
|
|
|
|
return e
|
2018-06-19 21:35:25 +00:00
|
|
|
|
2018-10-26 08:53:11 +00:00
|
|
|
def test_shaders_helper(stats, backend, args):
|
2018-06-19 21:35:25 +00:00
|
|
|
all_files = []
|
|
|
|
|
for root, dirs, files in os.walk(os.path.join(args.folder)):
|
2017-01-31 03:55:21 +00:00
|
|
|
files = [ f for f in files if not f.startswith(".") ] #ignore system files (esp OSX)
|
2016-05-11 17:39:38 +00:00
|
|
|
for i in files:
|
|
|
|
|
path = os.path.join(root, i)
|
2018-06-19 21:35:25 +00:00
|
|
|
relpath = os.path.relpath(path, args.folder)
|
|
|
|
|
all_files.append(relpath)
|
|
|
|
|
|
2019-11-05 15:59:41 +00:00
|
|
|
# The child processes in parallel execution mode don't have the proper state for the global args variable, so
|
2018-06-19 21:35:25 +00:00
|
|
|
# at this point we need to switch to explicit arguments
|
|
|
|
|
if args.parallel:
|
2020-04-27 14:44:14 +00:00
|
|
|
with multiprocessing.Pool(multiprocessing.cpu_count()) as pool:
|
|
|
|
|
results = []
|
|
|
|
|
for f in all_files:
|
|
|
|
|
results.append(pool.apply_async(test_shader_file,
|
|
|
|
|
args = (f, stats, args, backend)))
|
|
|
|
|
|
|
|
|
|
pool.close()
|
|
|
|
|
pool.join()
|
|
|
|
|
results_completed = [res.get() for res in results]
|
|
|
|
|
|
|
|
|
|
for error in results_completed:
|
|
|
|
|
if error is not None:
|
|
|
|
|
print('Error:', error)
|
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
2018-06-19 21:35:25 +00:00
|
|
|
else:
|
|
|
|
|
for i in all_files:
|
2019-03-07 11:43:00 +00:00
|
|
|
e = test_shader_file(i, stats, args, backend)
|
2019-01-14 10:23:46 +00:00
|
|
|
if e is not None:
|
|
|
|
|
print('Error:', e)
|
|
|
|
|
sys.exit(1)
|
2016-05-11 17:39:38 +00:00
|
|
|
|
2018-10-26 08:53:11 +00:00
|
|
|
def test_shaders(backend, args):
|
2018-06-19 21:35:25 +00:00
|
|
|
if args.malisc:
|
2016-03-22 13:44:12 +00:00
|
|
|
with open('stats.csv', 'w') as stats:
|
|
|
|
|
print('Shader,OrigRegs,OrigUniRegs,OrigALUShort,OrigLSShort,OrigTEXShort,OrigALULong,OrigLSLong,OrigTEXLong,CrossRegs,CrossUniRegs,CrossALUShort,CrossLSShort,CrossTEXShort,CrossALULong,CrossLSLong,CrossTEXLong', file = stats)
|
2018-10-26 08:53:11 +00:00
|
|
|
test_shaders_helper(stats, backend, args)
|
2016-03-22 13:44:12 +00:00
|
|
|
else:
|
2018-10-26 08:53:11 +00:00
|
|
|
test_shaders_helper(None, backend, args)
|
2016-03-22 13:44:12 +00:00
|
|
|
|
|
|
|
|
def main():
|
|
|
|
|
parser = argparse.ArgumentParser(description = 'Script for regression testing.')
|
|
|
|
|
parser.add_argument('folder',
|
|
|
|
|
help = 'Folder containing shader files to test.')
|
|
|
|
|
parser.add_argument('--update',
|
|
|
|
|
action = 'store_true',
|
|
|
|
|
help = 'Updates reference files if there is a mismatch. Use when legitimate changes in output is found.')
|
2016-03-22 13:56:50 +00:00
|
|
|
parser.add_argument('--keep',
|
|
|
|
|
action = 'store_true',
|
|
|
|
|
help = 'Leave failed GLSL shaders on disk if they fail regression. Useful for debugging.')
|
2021-12-09 16:29:12 +00:00
|
|
|
parser.add_argument('--diff',
|
|
|
|
|
action = 'store_true',
|
|
|
|
|
help = 'Displays a diff instead of the generated output on failure. Useful for debugging.')
|
2016-03-22 13:44:12 +00:00
|
|
|
parser.add_argument('--malisc',
|
|
|
|
|
action = 'store_true',
|
2016-04-04 07:36:04 +00:00
|
|
|
help = 'Use malisc offline compiler to determine static cycle counts before and after spirv-cross.')
|
2017-03-20 01:06:21 +00:00
|
|
|
parser.add_argument('--msl',
|
2017-01-18 18:05:57 +00:00
|
|
|
action = 'store_true',
|
|
|
|
|
help = 'Test Metal backend.')
|
2017-03-20 01:46:06 +00:00
|
|
|
parser.add_argument('--metal',
|
|
|
|
|
action = 'store_true',
|
|
|
|
|
help = 'Deprecated Metal option. Use --msl instead.')
|
2017-01-26 08:45:17 +00:00
|
|
|
parser.add_argument('--hlsl',
|
|
|
|
|
action = 'store_true',
|
|
|
|
|
help = 'Test HLSL backend.')
|
2017-02-05 10:04:45 +00:00
|
|
|
parser.add_argument('--force-no-external-validation',
|
|
|
|
|
action = 'store_true',
|
|
|
|
|
help = 'Disable all external validation.')
|
2017-11-21 08:52:53 +00:00
|
|
|
parser.add_argument('--opt',
|
|
|
|
|
action = 'store_true',
|
|
|
|
|
help = 'Run SPIRV-Tools optimization passes as well.')
|
2018-06-03 18:16:37 +00:00
|
|
|
parser.add_argument('--reflect',
|
|
|
|
|
action = 'store_true',
|
|
|
|
|
help = 'Test reflection backend.')
|
2018-06-19 21:35:25 +00:00
|
|
|
parser.add_argument('--parallel',
|
|
|
|
|
action = 'store_true',
|
|
|
|
|
help = 'Execute tests in parallel. Useful for doing regression quickly, but bad for debugging and stat output.')
|
2019-04-12 12:44:24 +00:00
|
|
|
parser.add_argument('--spirv-cross',
|
|
|
|
|
default = './spirv-cross',
|
|
|
|
|
help = 'Explicit path to spirv-cross')
|
2019-03-07 11:36:16 +00:00
|
|
|
parser.add_argument('--glslang',
|
|
|
|
|
default = 'glslangValidator',
|
|
|
|
|
help = 'Explicit path to glslangValidator')
|
|
|
|
|
parser.add_argument('--spirv-as',
|
|
|
|
|
default = 'spirv-as',
|
|
|
|
|
help = 'Explicit path to spirv-as')
|
|
|
|
|
parser.add_argument('--spirv-val',
|
|
|
|
|
default = 'spirv-val',
|
|
|
|
|
help = 'Explicit path to spirv-val')
|
|
|
|
|
parser.add_argument('--spirv-opt',
|
|
|
|
|
default = 'spirv-opt',
|
|
|
|
|
help = 'Explicit path to spirv-opt')
|
2019-04-08 09:53:13 +00:00
|
|
|
parser.add_argument('--iterations',
|
|
|
|
|
default = 1,
|
|
|
|
|
type = int,
|
|
|
|
|
help = 'Number of iterations to run SPIRV-Cross (benchmarking)')
|
2019-11-05 15:59:41 +00:00
|
|
|
|
2016-03-22 13:44:12 +00:00
|
|
|
args = parser.parse_args()
|
|
|
|
|
if not args.folder:
|
|
|
|
|
sys.stderr.write('Need shader folder.\n')
|
|
|
|
|
sys.exit(1)
|
|
|
|
|
|
2018-06-20 17:25:38 +00:00
|
|
|
if (args.parallel and (args.malisc or args.force_no_external_validation or args.update)):
|
|
|
|
|
sys.stderr.write('Parallel execution is disabled when using the flags --update, --malisc or --force-no-external-validation\n')
|
2018-06-19 21:35:25 +00:00
|
|
|
args.parallel = False
|
2019-11-05 15:59:41 +00:00
|
|
|
|
2019-06-18 10:22:46 +00:00
|
|
|
args.msl22 = False
|
2020-09-29 04:07:55 +00:00
|
|
|
args.msl23 = False
|
2021-10-21 09:46:45 +00:00
|
|
|
args.msl24 = False
|
2023-02-18 11:10:58 +00:00
|
|
|
args.msl30 = False
|
2023-11-28 13:25:51 +00:00
|
|
|
args.msl31 = False
|
2017-03-20 01:06:21 +00:00
|
|
|
if args.msl:
|
2017-01-31 03:55:21 +00:00
|
|
|
print_msl_compiler_version()
|
2023-02-20 12:41:28 +00:00
|
|
|
args.msl22 = msl_compiler_supports_version('-std=macos-metal2.2')
|
|
|
|
|
args.msl23 = msl_compiler_supports_version('-std=macos-metal2.3')
|
|
|
|
|
args.msl24 = msl_compiler_supports_version('-std=macos-metal2.4')
|
|
|
|
|
args.msl30 = msl_compiler_supports_version('-std=metal3.0')
|
2023-11-28 13:25:51 +00:00
|
|
|
args.msl31 = msl_compiler_supports_version('-std=metal3.1')
|
2017-01-25 08:01:53 +00:00
|
|
|
|
2018-10-26 08:53:11 +00:00
|
|
|
backend = 'glsl'
|
2019-11-05 15:59:41 +00:00
|
|
|
if (args.msl or args.metal):
|
2018-06-03 18:16:37 +00:00
|
|
|
backend = 'msl'
|
2019-11-05 15:59:41 +00:00
|
|
|
elif args.hlsl:
|
2018-06-03 18:16:37 +00:00
|
|
|
backend = 'hlsl'
|
|
|
|
|
elif args.reflect:
|
|
|
|
|
backend = 'reflect'
|
|
|
|
|
|
2018-10-26 08:53:11 +00:00
|
|
|
test_shaders(backend, args)
|
2016-03-22 13:44:12 +00:00
|
|
|
if args.malisc:
|
|
|
|
|
print('Stats in stats.csv!')
|
2016-03-22 13:56:50 +00:00
|
|
|
print('Tests completed!')
|
2016-03-02 17:09:16 +00:00
|
|
|
|
|
|
|
|
if __name__ == '__main__':
|
2016-03-22 13:44:12 +00:00
|
|
|
main()
|
