Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 1 | #!/usr/bin/env python3 |
| 2 | |
Hans-Kristian Arntzen | f9818f0 | 2020-01-16 15:24:37 +0100 | [diff] [blame] | 3 | # Copyright 2015-2020 Arm Limited |
| 4 | # |
| 5 | # Licensed under the Apache License, Version 2.0 (the "License"); |
| 6 | # you may not use this file except in compliance with the License. |
| 7 | # You may obtain a copy of the License at |
| 8 | # |
| 9 | # http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | # |
| 11 | # Unless required by applicable law or agreed to in writing, software |
| 12 | # distributed under the License is distributed on an "AS IS" BASIS, |
| 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 14 | # See the License for the specific language governing permissions and |
| 15 | # limitations under the License. |
Sarah Mashayekhi | eed8468 | 2019-11-26 14:46:19 -0500 | [diff] [blame] | 16 | |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 17 | import sys |
| 18 | import os |
Arseny Kapoulkine | f45075b | 2017-01-24 22:26:39 -0800 | [diff] [blame] | 19 | import os.path |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 20 | import subprocess |
| 21 | import tempfile |
| 22 | import re |
| 23 | import itertools |
| 24 | import hashlib |
| 25 | import shutil |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 26 | import argparse |
Hans-Kristian Arntzen | ad0c5c6 | 2017-02-04 10:19:44 +0100 | [diff] [blame] | 27 | import codecs |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 28 | import json |
Brad Davis | d0a67ba | 2018-06-19 14:35:25 -0700 | [diff] [blame] | 29 | import multiprocessing |
Hans-Kristian Arntzen | 26107ba | 2018-07-02 13:12:58 +0200 | [diff] [blame] | 30 | import errno |
Brad Davis | d0a67ba | 2018-06-19 14:35:25 -0700 | [diff] [blame] | 31 | from functools import partial |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 32 | |
Hans-Kristian Arntzen | 074f765 | 2019-03-07 12:36:16 +0100 | [diff] [blame] | 33 | class Paths(): |
Hans-Kristian Arntzen | 3a7cb0c | 2019-04-12 14:44:24 +0200 | [diff] [blame] | 34 | def __init__(self, spirv_cross, glslang, spirv_as, spirv_val, spirv_opt): |
| 35 | self.spirv_cross = spirv_cross |
Hans-Kristian Arntzen | 074f765 | 2019-03-07 12:36:16 +0100 | [diff] [blame] | 36 | self.glslang = glslang |
| 37 | self.spirv_as = spirv_as |
| 38 | self.spirv_val = spirv_val |
| 39 | self.spirv_opt = spirv_opt |
| 40 | |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 41 | def remove_file(path): |
| 42 | #print('Removing file:', path) |
| 43 | os.remove(path) |
| 44 | |
| 45 | def create_temporary(suff = ''): |
| 46 | f, path = tempfile.mkstemp(suffix = suff) |
| 47 | os.close(f) |
| 48 | #print('Creating temporary:', path) |
| 49 | return path |
| 50 | |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 51 | def parse_stats(stats): |
| 52 | m = re.search('([0-9]+) work registers', stats) |
| 53 | registers = int(m.group(1)) if m else 0 |
| 54 | |
| 55 | m = re.search('([0-9]+) uniform registers', stats) |
| 56 | uniform_regs = int(m.group(1)) if m else 0 |
| 57 | |
| 58 | m_list = re.findall('(-?[0-9]+)\s+(-?[0-9]+)\s+(-?[0-9]+)', stats) |
| 59 | alu_short = float(m_list[1][0]) if m_list else 0 |
| 60 | ls_short = float(m_list[1][1]) if m_list else 0 |
| 61 | tex_short = float(m_list[1][2]) if m_list else 0 |
| 62 | alu_long = float(m_list[2][0]) if m_list else 0 |
| 63 | ls_long = float(m_list[2][1]) if m_list else 0 |
| 64 | tex_long = float(m_list[2][2]) if m_list else 0 |
| 65 | |
| 66 | return (registers, uniform_regs, alu_short, ls_short, tex_short, alu_long, ls_long, tex_long) |
| 67 | |
| 68 | def get_shader_type(shader): |
| 69 | _, ext = os.path.splitext(shader) |
| 70 | if ext == '.vert': |
| 71 | return '--vertex' |
| 72 | elif ext == '.frag': |
| 73 | return '--fragment' |
| 74 | elif ext == '.comp': |
| 75 | return '--compute' |
| 76 | elif ext == '.tesc': |
| 77 | return '--tessellation_control' |
| 78 | elif ext == '.tese': |
| 79 | return '--tessellation_evaluation' |
| 80 | elif ext == '.geom': |
| 81 | return '--geometry' |
| 82 | else: |
| 83 | return '' |
| 84 | |
| 85 | def get_shader_stats(shader): |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 86 | path = create_temporary() |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 87 | |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 88 | p = subprocess.Popen(['malisc', get_shader_type(shader), '--core', 'Mali-T760', '-V', shader], stdout = subprocess.PIPE, stderr = subprocess.PIPE) |
| 89 | stdout, stderr = p.communicate() |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 90 | remove_file(path) |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 91 | |
| 92 | if p.returncode != 0: |
| 93 | print(stderr.decode('utf-8')) |
| 94 | raise OSError('malisc failed') |
| 95 | p.wait() |
| 96 | |
| 97 | returned = stdout.decode('utf-8') |
| 98 | return parse_stats(returned) |
| 99 | |
Bill Hollings | be4cb17 | 2017-01-30 22:55:21 -0500 | [diff] [blame] | 100 | def print_msl_compiler_version(): |
| 101 | try: |
| 102 | subprocess.check_call(['xcrun', '--sdk', 'iphoneos', 'metal', '--version']) |
| 103 | print('...are the Metal compiler characteristics.\n') # display after so xcrun FNF is silent |
| 104 | except OSError as e: |
Hans-Kristian Arntzen | 26107ba | 2018-07-02 13:12:58 +0200 | [diff] [blame] | 105 | if (e.errno != errno.ENOENT): # Ignore xcrun not found error |
Bill Hollings | be4cb17 | 2017-01-30 22:55:21 -0500 | [diff] [blame] | 106 | raise |
Hans-Kristian Arntzen | be1e27a | 2019-06-17 16:06:14 +0200 | [diff] [blame] | 107 | except subprocess.CalledProcessError: |
| 108 | pass |
| 109 | |
| 110 | def msl_compiler_supports_22(): |
| 111 | try: |
| 112 | subprocess.check_call(['xcrun', '--sdk', 'macosx', 'metal', '-x', 'metal', '-std=macos-metal2.2', '-'], |
| 113 | stdin = subprocess.DEVNULL, stdout = subprocess.DEVNULL, stderr = subprocess.DEVNULL) |
| 114 | print('Current SDK supports MSL 2.2. Enabling validation for MSL 2.2 shaders.') |
| 115 | return True |
| 116 | except OSError as e: |
| 117 | print('Failed to check if MSL 2.2 is not supported. It probably is not.') |
| 118 | return False |
| 119 | except subprocess.CalledProcessError: |
| 120 | print('Current SDK does NOT support MSL 2.2. Disabling validation for MSL 2.2 shaders.') |
| 121 | return False |
Bill Hollings | be4cb17 | 2017-01-30 22:55:21 -0500 | [diff] [blame] | 122 | |
Hans-Kristian Arntzen | 32823b0 | 2018-09-07 09:33:34 +0200 | [diff] [blame] | 123 | def path_to_msl_standard(shader): |
Chip Davis | 3a9af96 | 2018-09-26 20:06:05 -0500 | [diff] [blame] | 124 | if '.ios.' in shader: |
| 125 | if '.msl2.' in shader: |
| 126 | return '-std=ios-metal2.0' |
| 127 | elif '.msl21.' in shader: |
| 128 | return '-std=ios-metal2.1' |
Hans-Kristian Arntzen | fccf1d0 | 2019-06-12 09:30:41 +0200 | [diff] [blame] | 129 | elif '.msl22.' in shader: |
| 130 | return '-std=ios-metal2.2' |
Chip Davis | 3a9af96 | 2018-09-26 20:06:05 -0500 | [diff] [blame] | 131 | elif '.msl11.' in shader: |
| 132 | return '-std=ios-metal1.1' |
| 133 | elif '.msl10.' in shader: |
| 134 | return '-std=ios-metal1.0' |
| 135 | else: |
| 136 | return '-std=ios-metal1.2' |
Hans-Kristian Arntzen | 32823b0 | 2018-09-07 09:33:34 +0200 | [diff] [blame] | 137 | else: |
Chip Davis | 3a9af96 | 2018-09-26 20:06:05 -0500 | [diff] [blame] | 138 | if '.msl2.' in shader: |
| 139 | return '-std=macos-metal2.0' |
| 140 | elif '.msl21.' in shader: |
| 141 | return '-std=macos-metal2.1' |
Hans-Kristian Arntzen | fccf1d0 | 2019-06-12 09:30:41 +0200 | [diff] [blame] | 142 | elif '.msl22.' in shader: |
| 143 | return '-std=macos-metal2.2' |
Chip Davis | 3a9af96 | 2018-09-26 20:06:05 -0500 | [diff] [blame] | 144 | elif '.msl11.' in shader: |
| 145 | return '-std=macos-metal1.1' |
| 146 | else: |
| 147 | return '-std=macos-metal1.2' |
Hans-Kristian Arntzen | 32823b0 | 2018-09-07 09:33:34 +0200 | [diff] [blame] | 148 | |
Hans-Kristian Arntzen | 652d826 | 2018-09-07 09:45:25 +0200 | [diff] [blame] | 149 | def path_to_msl_standard_cli(shader): |
| 150 | if '.msl2.' in shader: |
| 151 | return '20000' |
| 152 | elif '.msl21.' in shader: |
| 153 | return '20100' |
Hans-Kristian Arntzen | fccf1d0 | 2019-06-12 09:30:41 +0200 | [diff] [blame] | 154 | elif '.msl22.' in shader: |
| 155 | return '20200' |
Hans-Kristian Arntzen | 652d826 | 2018-09-07 09:45:25 +0200 | [diff] [blame] | 156 | elif '.msl11.' in shader: |
| 157 | return '10100' |
| 158 | else: |
| 159 | return '10200' |
| 160 | |
Hans-Kristian Arntzen | 230a012 | 2017-11-21 09:52:53 +0100 | [diff] [blame] | 161 | def validate_shader_msl(shader, opt): |
| 162 | msl_path = reference_path(shader[0], shader[1], opt) |
Bill Hollings | be4cb17 | 2017-01-30 22:55:21 -0500 | [diff] [blame] | 163 | try: |
Chip Davis | 3a9af96 | 2018-09-26 20:06:05 -0500 | [diff] [blame] | 164 | if '.ios.' in msl_path: |
| 165 | msl_os = 'iphoneos' |
| 166 | else: |
| 167 | msl_os = 'macosx' |
Hans-Kristian Arntzen | 32823b0 | 2018-09-07 09:33:34 +0200 | [diff] [blame] | 168 | subprocess.check_call(['xcrun', '--sdk', msl_os, 'metal', '-x', 'metal', path_to_msl_standard(msl_path), '-Werror', '-Wno-unused-variable', msl_path]) |
Bill Hollings | be4cb17 | 2017-01-30 22:55:21 -0500 | [diff] [blame] | 169 | print('Compiled Metal shader: ' + msl_path) # display after so xcrun FNF is silent |
| 170 | except OSError as oe: |
Hans-Kristian Arntzen | 26107ba | 2018-07-02 13:12:58 +0200 | [diff] [blame] | 171 | if (oe.errno != errno.ENOENT): # Ignore xcrun not found error |
Bill Hollings | be4cb17 | 2017-01-30 22:55:21 -0500 | [diff] [blame] | 172 | raise |
| 173 | except subprocess.CalledProcessError: |
| 174 | print('Error compiling Metal shader: ' + msl_path) |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 175 | raise RuntimeError('Failed to compile Metal shader') |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 176 | |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 177 | def cross_compile_msl(shader, spirv, opt, iterations, paths): |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 178 | spirv_path = create_temporary() |
| 179 | msl_path = create_temporary(os.path.basename(shader)) |
Hans-Kristian Arntzen | db5d49f | 2017-09-29 11:07:11 +0200 | [diff] [blame] | 180 | |
Hans-Kristian Arntzen | 9012a39 | 2020-01-06 11:47:26 +0100 | [diff] [blame] | 181 | spirv_env = 'vulkan1.1spv1.4' if ('.spv14.' in shader) else 'vulkan1.1' |
| 182 | |
| 183 | spirv_cmd = [paths.spirv_as, '--target-env', spirv_env, '-o', spirv_path, shader] |
Hans-Kristian Arntzen | d2cc43e | 2019-02-19 17:00:49 +0100 | [diff] [blame] | 184 | if '.preserve.' in shader: |
| 185 | spirv_cmd.append('--preserve-numeric-ids') |
| 186 | |
Hans-Kristian Arntzen | db5d49f | 2017-09-29 11:07:11 +0200 | [diff] [blame] | 187 | if spirv: |
Hans-Kristian Arntzen | d2cc43e | 2019-02-19 17:00:49 +0100 | [diff] [blame] | 188 | subprocess.check_call(spirv_cmd) |
Hans-Kristian Arntzen | db5d49f | 2017-09-29 11:07:11 +0200 | [diff] [blame] | 189 | else: |
Ryan Harrison | cf1bf1c | 2019-09-18 15:56:51 -0400 | [diff] [blame] | 190 | subprocess.check_call([paths.glslang, '--amb' ,'--target-env', 'vulkan1.1', '-V', '-o', spirv_path, shader]) |
Hans-Kristian Arntzen | db5d49f | 2017-09-29 11:07:11 +0200 | [diff] [blame] | 191 | |
Dan Sinclair | e5af412 | 2019-11-05 10:59:41 -0500 | [diff] [blame] | 192 | if opt and (not shader_is_invalid_spirv(shader)): |
Lukas Hermanns | e1b161b | 2019-10-21 16:39:53 -0400 | [diff] [blame] | 193 | if '.graphics-robust-access.' in shader: |
| 194 | subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '--graphics-robust-access', '-o', spirv_path, spirv_path]) |
| 195 | else: |
| 196 | subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '-o', spirv_path, spirv_path]) |
Hans-Kristian Arntzen | 230a012 | 2017-11-21 09:52:53 +0100 | [diff] [blame] | 197 | |
Hans-Kristian Arntzen | 3a7cb0c | 2019-04-12 14:44:24 +0200 | [diff] [blame] | 198 | spirv_cross_path = paths.spirv_cross |
Hans-Kristian Arntzen | e8ca39b | 2018-04-03 14:26:24 +0200 | [diff] [blame] | 199 | |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 200 | msl_args = [spirv_cross_path, '--entry', 'main', '--output', msl_path, spirv_path, '--msl', '--iterations', str(iterations)] |
Hans-Kristian Arntzen | 652d826 | 2018-09-07 09:45:25 +0200 | [diff] [blame] | 201 | msl_args.append('--msl-version') |
| 202 | msl_args.append(path_to_msl_standard_cli(shader)) |
Chip Davis | 2583321 | 2018-09-19 20:36:33 -0500 | [diff] [blame] | 203 | if '.swizzle.' in shader: |
| 204 | msl_args.append('--msl-swizzle-texture-samples') |
Chip Davis | 3a9af96 | 2018-09-26 20:06:05 -0500 | [diff] [blame] | 205 | if '.ios.' in shader: |
| 206 | msl_args.append('--msl-ios') |
Hans-Kristian Arntzen | b8033d7 | 2019-01-14 14:53:47 +0100 | [diff] [blame] | 207 | if '.pad-fragment.' in shader: |
| 208 | msl_args.append('--msl-pad-fragment-output') |
Chip Davis | c51e5b7 | 2019-01-08 16:33:32 -0600 | [diff] [blame] | 209 | if '.capture.' in shader: |
| 210 | msl_args.append('--msl-capture-output') |
Chip Davis | 41d9424 | 2019-02-05 23:47:50 -0600 | [diff] [blame] | 211 | if '.domain.' in shader: |
| 212 | msl_args.append('--msl-domain-lower-left') |
Hans-Kristian Arntzen | e47a77d | 2019-03-14 10:29:34 +0100 | [diff] [blame] | 213 | if '.argument.' in shader: |
| 214 | msl_args.append('--msl-argument-buffers') |
Hans-Kristian Arntzen | fc4f39b | 2019-04-23 12:17:21 +0200 | [diff] [blame] | 215 | if '.texture-buffer-native.' in shader: |
| 216 | msl_args.append('--msl-texture-buffer-native') |
Lukas Hermanns | f3a6d28 | 2019-09-27 15:49:54 -0400 | [diff] [blame] | 217 | if '.framebuffer-fetch.' in shader: |
| 218 | msl_args.append('--msl-framebuffer-fetch') |
Lukas Hermanns | ffbd801 | 2019-10-09 11:22:25 -0400 | [diff] [blame] | 219 | if '.invariant-float-math.' in shader: |
| 220 | msl_args.append('--msl-invariant-float-math') |
Lukas Hermanns | f3a6d28 | 2019-09-27 15:49:54 -0400 | [diff] [blame] | 221 | if '.emulate-cube-array.' in shader: |
| 222 | msl_args.append('--msl-emulate-cube-array') |
Hans-Kristian Arntzen | e2aadf8 | 2019-03-15 21:53:21 +0100 | [diff] [blame] | 223 | if '.discrete.' in shader: |
Hans-Kristian Arntzen | b3380ec | 2019-03-15 14:07:03 +0100 | [diff] [blame] | 224 | # Arbitrary for testing purposes. |
Hans-Kristian Arntzen | e2aadf8 | 2019-03-15 21:53:21 +0100 | [diff] [blame] | 225 | msl_args.append('--msl-discrete-descriptor-set') |
Hans-Kristian Arntzen | b3380ec | 2019-03-15 14:07:03 +0100 | [diff] [blame] | 226 | msl_args.append('2') |
Hans-Kristian Arntzen | e2aadf8 | 2019-03-15 21:53:21 +0100 | [diff] [blame] | 227 | msl_args.append('--msl-discrete-descriptor-set') |
Hans-Kristian Arntzen | b3380ec | 2019-03-15 14:07:03 +0100 | [diff] [blame] | 228 | msl_args.append('3') |
Hans-Kristian Arntzen | c3bd136 | 2020-01-16 11:07:30 +0100 | [diff] [blame] | 229 | if '.force-active.' in shader: |
| 230 | msl_args.append('--msl-force-active-argument-buffer-resources') |
Hans-Kristian Arntzen | 65af09d | 2019-05-28 13:41:46 +0200 | [diff] [blame] | 231 | if '.line.' in shader: |
| 232 | msl_args.append('--emit-line-directives') |
Chip Davis | 7eecf5a | 2019-05-31 12:06:20 -0500 | [diff] [blame] | 233 | if '.multiview.' in shader: |
| 234 | msl_args.append('--msl-multiview') |
Chip Davis | 6a58554 | 2019-07-12 21:50:50 -0500 | [diff] [blame] | 235 | if '.viewfromdev.' in shader: |
| 236 | msl_args.append('--msl-view-index-from-device-index') |
Chip Davis | fb5ee4c | 2019-07-22 13:08:04 -0500 | [diff] [blame] | 237 | if '.dispatchbase.' in shader: |
| 238 | msl_args.append('--msl-dispatch-base') |
Chip Davis | cb35934 | 2019-09-05 23:14:12 -0500 | [diff] [blame] | 239 | if '.dynamic-buffer.' in shader: |
| 240 | # Arbitrary for testing purposes. |
| 241 | msl_args.append('--msl-dynamic-buffer') |
| 242 | msl_args.append('0') |
| 243 | msl_args.append('0') |
| 244 | msl_args.append('--msl-dynamic-buffer') |
| 245 | msl_args.append('1') |
| 246 | msl_args.append('2') |
Chip Davis | fedbc35 | 2019-12-16 22:58:16 -0600 | [diff] [blame] | 247 | if '.inline-block.' in shader: |
| 248 | # Arbitrary for testing purposes. |
| 249 | msl_args.append('--msl-inline-uniform-block') |
| 250 | msl_args.append('0') |
| 251 | msl_args.append('0') |
Hans-Kristian Arntzen | 4bb673a | 2019-10-14 12:51:48 +0200 | [diff] [blame] | 252 | if '.device-argument-buffer.' in shader: |
| 253 | msl_args.append('--msl-device-argument-buffer') |
| 254 | msl_args.append('0') |
| 255 | msl_args.append('--msl-device-argument-buffer') |
| 256 | msl_args.append('1') |
Hans-Kristian Arntzen | c9d4f9c | 2020-02-24 12:47:14 +0100 | [diff] [blame] | 257 | if '.force-native-array.' in shader: |
| 258 | msl_args.append('--msl-force-native-arrays') |
Hans-Kristian Arntzen | b8905bb | 2020-03-26 11:21:23 +0100 | [diff] [blame] | 259 | if '.zero-initialize.' in shader: |
| 260 | msl_args.append('--force-zero-initialized-variables') |
Chip Davis | b29f83c | 2020-04-10 01:13:33 -0500 | [diff] [blame] | 261 | if '.frag-output.' in shader: |
| 262 | # Arbitrary for testing purposes. |
| 263 | msl_args.append('--msl-disable-frag-depth-builtin') |
| 264 | msl_args.append('--msl-disable-frag-stencil-ref-builtin') |
| 265 | msl_args.append('--msl-enable-frag-output-mask') |
| 266 | msl_args.append('0x000000ca') |
Hans-Kristian Arntzen | ebf4636 | 2020-04-20 09:48:20 +0200 | [diff] [blame] | 267 | if '.no-user-varying.' in shader: |
| 268 | msl_args.append('--msl-no-clip-distance-user-varying') |
Chip Davis | 5281d99 | 2020-06-13 23:03:30 -0500 | [diff] [blame] | 269 | if '.shader-inputs.' in shader: |
| 270 | # Arbitrary for testing purposes. |
| 271 | msl_args.append('--msl-shader-input') |
| 272 | msl_args.append('0') |
| 273 | msl_args.append('u8') |
| 274 | msl_args.append('2') |
| 275 | msl_args.append('--msl-shader-input') |
| 276 | msl_args.append('1') |
| 277 | msl_args.append('u16') |
| 278 | msl_args.append('3') |
| 279 | msl_args.append('--msl-shader-input') |
| 280 | msl_args.append('6') |
| 281 | msl_args.append('other') |
| 282 | msl_args.append('4') |
Hans-Kristian Arntzen | e8ca39b | 2018-04-03 14:26:24 +0200 | [diff] [blame] | 283 | |
| 284 | subprocess.check_call(msl_args) |
Hans-Kristian Arntzen | 146ea76 | 2018-04-11 10:28:39 +0200 | [diff] [blame] | 285 | |
| 286 | if not shader_is_invalid_spirv(msl_path): |
Hans-Kristian Arntzen | 9012a39 | 2020-01-06 11:47:26 +0100 | [diff] [blame] | 287 | subprocess.check_call([paths.spirv_val, '--scalar-block-layout', '--target-env', spirv_env, spirv_path]) |
Hans-Kristian Arntzen | 146ea76 | 2018-04-11 10:28:39 +0200 | [diff] [blame] | 288 | |
Hans-Kristian Arntzen | 1c28ec6 | 2017-01-18 19:05:57 +0100 | [diff] [blame] | 289 | return (spirv_path, msl_path) |
| 290 | |
Hans-Kristian Arntzen | 9507325 | 2017-12-12 11:03:46 +0100 | [diff] [blame] | 291 | def shader_model_hlsl(shader): |
| 292 | if '.vert' in shader: |
Hans-Kristian Arntzen | 0312c98 | 2018-09-12 09:44:35 +0200 | [diff] [blame] | 293 | if '.sm30.' in shader: |
| 294 | return '-Tvs_3_0' |
| 295 | else: |
| 296 | return '-Tvs_5_1' |
Hans-Kristian Arntzen | 9507325 | 2017-12-12 11:03:46 +0100 | [diff] [blame] | 297 | elif '.frag' in shader: |
Hans-Kristian Arntzen | 0312c98 | 2018-09-12 09:44:35 +0200 | [diff] [blame] | 298 | if '.sm30.' in shader: |
| 299 | return '-Tps_3_0' |
| 300 | else: |
| 301 | return '-Tps_5_1' |
Hans-Kristian Arntzen | 9507325 | 2017-12-12 11:03:46 +0100 | [diff] [blame] | 302 | elif '.comp' in shader: |
| 303 | return '-Tcs_5_1' |
| 304 | else: |
| 305 | return None |
| 306 | |
Hans-Kristian Arntzen | 789fa91 | 2017-12-12 13:23:56 +0100 | [diff] [blame] | 307 | def shader_to_win_path(shader): |
| 308 | # It's (very) convenient to be able to run HLSL testing in wine on Unix-likes, so support that. |
| 309 | try: |
| 310 | with subprocess.Popen(['winepath', '-w', shader], stdout = subprocess.PIPE, stderr = subprocess.PIPE) as f: |
| 311 | stdout_data, stderr_data = f.communicate() |
| 312 | return stdout_data.decode('utf-8') |
| 313 | except OSError as oe: |
Hans-Kristian Arntzen | 26107ba | 2018-07-02 13:12:58 +0200 | [diff] [blame] | 314 | if (oe.errno != errno.ENOENT): # Ignore not found errors |
Hans-Kristian Arntzen | 789fa91 | 2017-12-12 13:23:56 +0100 | [diff] [blame] | 315 | return shader |
| 316 | except subprocess.CalledProcessError: |
| 317 | raise |
| 318 | |
Hans-Kristian Arntzen | 51a48a7 | 2017-12-12 13:33:13 +0100 | [diff] [blame] | 319 | return shader |
| 320 | |
Hans-Kristian Arntzen | d0c829b | 2018-04-18 16:43:28 +0200 | [diff] [blame] | 321 | ignore_fxc = False |
Hans-Kristian Arntzen | 074f765 | 2019-03-07 12:36:16 +0100 | [diff] [blame] | 322 | def validate_shader_hlsl(shader, force_no_external_validation, paths): |
Hans-Kristian Arntzen | c7eda1b | 2019-07-17 11:24:31 +0200 | [diff] [blame] | 323 | test_glslang = True |
| 324 | if '.nonuniformresource.' in shader: |
| 325 | test_glslang = False |
| 326 | if '.fxconly.' in shader: |
| 327 | test_glslang = False |
| 328 | |
Hans-Kristian Arntzen | 8f71694 | 2020-07-01 11:37:03 +0200 | [diff] [blame] | 329 | hlsl_args = [paths.glslang, '--amb', '-e', 'main', '-D', '--target-env', 'vulkan1.1', '-V', shader] |
| 330 | if '.sm30.' in shader: |
| 331 | hlsl_args.append('--hlsl-dx9-compatible') |
| 332 | |
Hans-Kristian Arntzen | c7eda1b | 2019-07-17 11:24:31 +0200 | [diff] [blame] | 333 | if test_glslang: |
Hans-Kristian Arntzen | 8f71694 | 2020-07-01 11:37:03 +0200 | [diff] [blame] | 334 | subprocess.check_call(hlsl_args) |
| 335 | |
Hans-Kristian Arntzen | 789fa91 | 2017-12-12 13:23:56 +0100 | [diff] [blame] | 336 | is_no_fxc = '.nofxc.' in shader |
Hans-Kristian Arntzen | d0c829b | 2018-04-18 16:43:28 +0200 | [diff] [blame] | 337 | global ignore_fxc |
Hans-Kristian Arntzen | 6157bf3 | 2018-10-26 10:53:11 +0200 | [diff] [blame] | 338 | if (not ignore_fxc) and (not force_no_external_validation) and (not is_no_fxc): |
Hans-Kristian Arntzen | 789fa91 | 2017-12-12 13:23:56 +0100 | [diff] [blame] | 339 | try: |
| 340 | win_path = shader_to_win_path(shader) |
Hans-Kristian Arntzen | 647ddae | 2019-05-13 14:58:27 +0200 | [diff] [blame] | 341 | args = ['fxc', '-nologo', shader_model_hlsl(shader), win_path] |
| 342 | if '.nonuniformresource.' in shader: |
| 343 | args.append('/enable_unbounded_descriptor_tables') |
| 344 | subprocess.check_call(args) |
Hans-Kristian Arntzen | 789fa91 | 2017-12-12 13:23:56 +0100 | [diff] [blame] | 345 | except OSError as oe: |
Hans-Kristian Arntzen | 26107ba | 2018-07-02 13:12:58 +0200 | [diff] [blame] | 346 | if (oe.errno != errno.ENOENT): # Ignore not found errors |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 347 | print('Failed to run FXC.') |
| 348 | ignore_fxc = True |
Hans-Kristian Arntzen | 789fa91 | 2017-12-12 13:23:56 +0100 | [diff] [blame] | 349 | raise |
Hans-Kristian Arntzen | d0c829b | 2018-04-18 16:43:28 +0200 | [diff] [blame] | 350 | else: |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 351 | print('Could not find FXC.') |
Hans-Kristian Arntzen | d0c829b | 2018-04-18 16:43:28 +0200 | [diff] [blame] | 352 | ignore_fxc = True |
Hans-Kristian Arntzen | 789fa91 | 2017-12-12 13:23:56 +0100 | [diff] [blame] | 353 | except subprocess.CalledProcessError: |
| 354 | print('Failed compiling HLSL shader:', shader, 'with FXC.') |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 355 | raise RuntimeError('Failed compiling HLSL shader') |
Robert Konrad | ddbeea2 | 2017-01-26 11:46:29 +0100 | [diff] [blame] | 356 | |
Hans-Kristian Arntzen | ecf56cf | 2017-06-30 10:34:21 +0200 | [diff] [blame] | 357 | def shader_to_sm(shader): |
Hans-Kristian Arntzen | 2d52006 | 2020-06-04 11:35:21 +0200 | [diff] [blame] | 358 | if '.sm62.' in shader: |
| 359 | return '62' |
| 360 | elif '.sm60.' in shader: |
Hans-Kristian Arntzen | c266429 | 2018-04-11 15:02:02 +0200 | [diff] [blame] | 361 | return '60' |
| 362 | elif '.sm51.' in shader: |
Hans-Kristian Arntzen | ecf56cf | 2017-06-30 10:34:21 +0200 | [diff] [blame] | 363 | return '51' |
crosire | 3d39652 | 2018-09-11 20:57:56 +0200 | [diff] [blame] | 364 | elif '.sm30.' in shader: |
| 365 | return '30' |
Hans-Kristian Arntzen | ecf56cf | 2017-06-30 10:34:21 +0200 | [diff] [blame] | 366 | else: |
| 367 | return '50' |
| 368 | |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 369 | def cross_compile_hlsl(shader, spirv, opt, force_no_external_validation, iterations, paths): |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 370 | spirv_path = create_temporary() |
| 371 | hlsl_path = create_temporary(os.path.basename(shader)) |
Hans-Kristian Arntzen | db5d49f | 2017-09-29 11:07:11 +0200 | [diff] [blame] | 372 | |
Ryan Harrison | d3c59e7 | 2019-05-08 14:03:35 -0400 | [diff] [blame] | 373 | spirv_cmd = [paths.spirv_as, '--target-env', 'vulkan1.1', '-o', spirv_path, shader] |
Hans-Kristian Arntzen | d2cc43e | 2019-02-19 17:00:49 +0100 | [diff] [blame] | 374 | if '.preserve.' in shader: |
| 375 | spirv_cmd.append('--preserve-numeric-ids') |
| 376 | |
Hans-Kristian Arntzen | db5d49f | 2017-09-29 11:07:11 +0200 | [diff] [blame] | 377 | if spirv: |
Hans-Kristian Arntzen | d2cc43e | 2019-02-19 17:00:49 +0100 | [diff] [blame] | 378 | subprocess.check_call(spirv_cmd) |
Hans-Kristian Arntzen | db5d49f | 2017-09-29 11:07:11 +0200 | [diff] [blame] | 379 | else: |
Ryan Harrison | cf1bf1c | 2019-09-18 15:56:51 -0400 | [diff] [blame] | 380 | subprocess.check_call([paths.glslang, '--amb', '--target-env', 'vulkan1.1', '-V', '-o', spirv_path, shader]) |
Hans-Kristian Arntzen | db5d49f | 2017-09-29 11:07:11 +0200 | [diff] [blame] | 381 | |
Dan Sinclair | e5af412 | 2019-11-05 10:59:41 -0500 | [diff] [blame] | 382 | if opt and (not shader_is_invalid_spirv(hlsl_path)): |
Hans-Kristian Arntzen | 074f765 | 2019-03-07 12:36:16 +0100 | [diff] [blame] | 383 | subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '-o', spirv_path, spirv_path]) |
Hans-Kristian Arntzen | 230a012 | 2017-11-21 09:52:53 +0100 | [diff] [blame] | 384 | |
Hans-Kristian Arntzen | 3a7cb0c | 2019-04-12 14:44:24 +0200 | [diff] [blame] | 385 | spirv_cross_path = paths.spirv_cross |
Hans-Kristian Arntzen | ecf56cf | 2017-06-30 10:34:21 +0200 | [diff] [blame] | 386 | |
| 387 | sm = shader_to_sm(shader) |
Hans-Kristian Arntzen | 65af09d | 2019-05-28 13:41:46 +0200 | [diff] [blame] | 388 | |
| 389 | hlsl_args = [spirv_cross_path, '--entry', 'main', '--output', hlsl_path, spirv_path, '--hlsl-enable-compat', '--hlsl', '--shader-model', sm, '--iterations', str(iterations)] |
| 390 | if '.line.' in shader: |
| 391 | hlsl_args.append('--emit-line-directives') |
Hans-Kristian Arntzen | c27e1ef | 2020-03-04 16:32:52 +0100 | [diff] [blame] | 392 | if '.force-uav.' in shader: |
| 393 | hlsl_args.append('--hlsl-force-storage-buffer-as-uav') |
Hans-Kristian Arntzen | b8905bb | 2020-03-26 11:21:23 +0100 | [diff] [blame] | 394 | if '.zero-initialize.' in shader: |
| 395 | hlsl_args.append('--force-zero-initialized-variables') |
Hans-Kristian Arntzen | 28bf905 | 2020-04-03 11:21:41 +0200 | [diff] [blame] | 396 | if '.nonwritable-uav-texture.' in shader: |
| 397 | hlsl_args.append('--hlsl-nonwritable-uav-texture-as-srv') |
Hans-Kristian Arntzen | 2d52006 | 2020-06-04 11:35:21 +0200 | [diff] [blame] | 398 | if '.native-16bit.' in shader: |
| 399 | hlsl_args.append('--hlsl-enable-16bit-types') |
Hans-Kristian Arntzen | b8905bb | 2020-03-26 11:21:23 +0100 | [diff] [blame] | 400 | |
Hans-Kristian Arntzen | 65af09d | 2019-05-28 13:41:46 +0200 | [diff] [blame] | 401 | subprocess.check_call(hlsl_args) |
Hans-Kristian Arntzen | 146ea76 | 2018-04-11 10:28:39 +0200 | [diff] [blame] | 402 | |
| 403 | if not shader_is_invalid_spirv(hlsl_path): |
Hans-Kristian Arntzen | 1721850 | 2019-07-22 11:30:03 +0200 | [diff] [blame] | 404 | subprocess.check_call([paths.spirv_val, '--scalar-block-layout', '--target-env', 'vulkan1.1', spirv_path]) |
Robert Konrad | cec9c70 | 2017-01-26 09:45:17 +0100 | [diff] [blame] | 405 | |
Hans-Kristian Arntzen | 074f765 | 2019-03-07 12:36:16 +0100 | [diff] [blame] | 406 | validate_shader_hlsl(hlsl_path, force_no_external_validation, paths) |
Dan Sinclair | e5af412 | 2019-11-05 10:59:41 -0500 | [diff] [blame] | 407 | |
Robert Konrad | e717953 | 2017-01-26 10:06:05 +0100 | [diff] [blame] | 408 | return (spirv_path, hlsl_path) |
Robert Konrad | cec9c70 | 2017-01-26 09:45:17 +0100 | [diff] [blame] | 409 | |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 410 | def cross_compile_reflect(shader, spirv, opt, iterations, paths): |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 411 | spirv_path = create_temporary() |
| 412 | reflect_path = create_temporary(os.path.basename(shader)) |
| 413 | |
Ryan Harrison | d3c59e7 | 2019-05-08 14:03:35 -0400 | [diff] [blame] | 414 | spirv_cmd = [paths.spirv_as, '--target-env', 'vulkan1.1', '-o', spirv_path, shader] |
Hans-Kristian Arntzen | d2cc43e | 2019-02-19 17:00:49 +0100 | [diff] [blame] | 415 | if '.preserve.' in shader: |
| 416 | spirv_cmd.append('--preserve-numeric-ids') |
| 417 | |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 418 | if spirv: |
Hans-Kristian Arntzen | d2cc43e | 2019-02-19 17:00:49 +0100 | [diff] [blame] | 419 | subprocess.check_call(spirv_cmd) |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 420 | else: |
Ryan Harrison | cf1bf1c | 2019-09-18 15:56:51 -0400 | [diff] [blame] | 421 | subprocess.check_call([paths.glslang, '--amb', '--target-env', 'vulkan1.1', '-V', '-o', spirv_path, shader]) |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 422 | |
Dan Sinclair | e5af412 | 2019-11-05 10:59:41 -0500 | [diff] [blame] | 423 | if opt and (not shader_is_invalid_spirv(reflect_path)): |
Hans-Kristian Arntzen | 074f765 | 2019-03-07 12:36:16 +0100 | [diff] [blame] | 424 | subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '-o', spirv_path, spirv_path]) |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 425 | |
Hans-Kristian Arntzen | 3a7cb0c | 2019-04-12 14:44:24 +0200 | [diff] [blame] | 426 | spirv_cross_path = paths.spirv_cross |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 427 | |
| 428 | sm = shader_to_sm(shader) |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 429 | subprocess.check_call([spirv_cross_path, '--entry', 'main', '--output', reflect_path, spirv_path, '--reflect', '--iterations', str(iterations)]) |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 430 | return (spirv_path, reflect_path) |
| 431 | |
Hans-Kristian Arntzen | 074f765 | 2019-03-07 12:36:16 +0100 | [diff] [blame] | 432 | def validate_shader(shader, vulkan, paths): |
Arseny Kapoulkine | f45075b | 2017-01-24 22:26:39 -0800 | [diff] [blame] | 433 | if vulkan: |
Ryan Harrison | cf1bf1c | 2019-09-18 15:56:51 -0400 | [diff] [blame] | 434 | subprocess.check_call([paths.glslang, '--amb', '--target-env', 'vulkan1.1', '-V', shader]) |
Arseny Kapoulkine | f45075b | 2017-01-24 22:26:39 -0800 | [diff] [blame] | 435 | else: |
Hans-Kristian Arntzen | 074f765 | 2019-03-07 12:36:16 +0100 | [diff] [blame] | 436 | subprocess.check_call([paths.glslang, shader]) |
Arseny Kapoulkine | f45075b | 2017-01-24 22:26:39 -0800 | [diff] [blame] | 437 | |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 438 | def cross_compile(shader, vulkan, spirv, invalid_spirv, eliminate, is_legacy, flatten_ubo, sso, flatten_dim, opt, push_ubo, iterations, paths): |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 439 | spirv_path = create_temporary() |
| 440 | glsl_path = create_temporary(os.path.basename(shader)) |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 441 | |
Hans-Kristian Arntzen | 9012a39 | 2020-01-06 11:47:26 +0100 | [diff] [blame] | 442 | spirv_env = 'vulkan1.1spv1.4' if ('.spv14.' in shader) else 'vulkan1.1' |
| 443 | |
Hans-Kristian Arntzen | 45ad58a | 2016-05-10 23:39:41 +0200 | [diff] [blame] | 444 | if vulkan or spirv: |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 445 | vulkan_glsl_path = create_temporary('vk' + os.path.basename(shader)) |
Hans-Kristian Arntzen | dbee4e4 | 2016-05-05 10:16:22 +0200 | [diff] [blame] | 446 | |
Hans-Kristian Arntzen | 9012a39 | 2020-01-06 11:47:26 +0100 | [diff] [blame] | 447 | spirv_cmd = [paths.spirv_as, '--target-env', spirv_env, '-o', spirv_path, shader] |
Hans-Kristian Arntzen | d2cc43e | 2019-02-19 17:00:49 +0100 | [diff] [blame] | 448 | if '.preserve.' in shader: |
| 449 | spirv_cmd.append('--preserve-numeric-ids') |
| 450 | |
Hans-Kristian Arntzen | 45ad58a | 2016-05-10 23:39:41 +0200 | [diff] [blame] | 451 | if spirv: |
Hans-Kristian Arntzen | d2cc43e | 2019-02-19 17:00:49 +0100 | [diff] [blame] | 452 | subprocess.check_call(spirv_cmd) |
Hans-Kristian Arntzen | 45ad58a | 2016-05-10 23:39:41 +0200 | [diff] [blame] | 453 | else: |
Ryan Harrison | cf1bf1c | 2019-09-18 15:56:51 -0400 | [diff] [blame] | 454 | subprocess.check_call([paths.glslang, '--amb', '--target-env', 'vulkan1.1', '-V', '-o', spirv_path, shader]) |
Hans-Kristian Arntzen | 45ad58a | 2016-05-10 23:39:41 +0200 | [diff] [blame] | 455 | |
Hans-Kristian Arntzen | cdca192 | 2017-11-21 16:52:01 +0100 | [diff] [blame] | 456 | if opt and (not invalid_spirv): |
Hans-Kristian Arntzen | 074f765 | 2019-03-07 12:36:16 +0100 | [diff] [blame] | 457 | subprocess.check_call([paths.spirv_opt, '--skip-validation', '-O', '-o', spirv_path, spirv_path]) |
Hans-Kristian Arntzen | 230a012 | 2017-11-21 09:52:53 +0100 | [diff] [blame] | 458 | |
Hans-Kristian Arntzen | 706d3ea | 2016-09-12 20:11:30 +0200 | [diff] [blame] | 459 | if not invalid_spirv: |
Hans-Kristian Arntzen | 9012a39 | 2020-01-06 11:47:26 +0100 | [diff] [blame] | 460 | subprocess.check_call([paths.spirv_val, '--scalar-block-layout', '--target-env', spirv_env, spirv_path]) |
Hans-Kristian Arntzen | dbee4e4 | 2016-05-05 10:16:22 +0200 | [diff] [blame] | 461 | |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 462 | extra_args = ['--iterations', str(iterations)] |
Arseny Kapoulkine | 24c6625 | 2017-01-16 14:19:49 -0800 | [diff] [blame] | 463 | if eliminate: |
| 464 | extra_args += ['--remove-unused-variables'] |
Hans-Kristian Arntzen | 41f7e5b | 2017-01-13 16:41:27 +0100 | [diff] [blame] | 465 | if is_legacy: |
Arseny Kapoulkine | 24c6625 | 2017-01-16 14:19:49 -0800 | [diff] [blame] | 466 | extra_args += ['--version', '100', '--es'] |
| 467 | if flatten_ubo: |
| 468 | extra_args += ['--flatten-ubo'] |
Hans-Kristian Arntzen | 36650c8 | 2017-05-22 15:30:43 +0200 | [diff] [blame] | 469 | if sso: |
| 470 | extra_args += ['--separate-shader-objects'] |
Hans-Kristian Arntzen | 470ae7a | 2017-05-22 17:40:00 +0200 | [diff] [blame] | 471 | if flatten_dim: |
| 472 | extra_args += ['--flatten-multidimensional-arrays'] |
Hans-Kristian Arntzen | 0474848 | 2019-03-19 10:58:37 +0100 | [diff] [blame] | 473 | if push_ubo: |
| 474 | extra_args += ['--glsl-emit-push-constant-as-ubo'] |
Hans-Kristian Arntzen | 65af09d | 2019-05-28 13:41:46 +0200 | [diff] [blame] | 475 | if '.line.' in shader: |
| 476 | extra_args += ['--emit-line-directives'] |
Hans-Kristian Arntzen | 12ca9d1 | 2019-07-25 11:07:14 +0200 | [diff] [blame] | 477 | if '.no-samplerless.' in shader: |
| 478 | extra_args += ['--vulkan-glsl-disable-ext-samplerless-texture-functions'] |
Hans-Kristian Arntzen | 95cd20f | 2020-03-04 16:41:00 +0100 | [diff] [blame] | 479 | if '.no-qualifier-deduction.' in shader: |
| 480 | extra_args += ['--disable-storage-image-qualifier-deduction'] |
Hans-Kristian Arntzen | 04e877d | 2020-03-19 14:20:37 +0100 | [diff] [blame] | 481 | if '.framebuffer-fetch.' in shader: |
| 482 | extra_args += ['--glsl-remap-ext-framebuffer-fetch', '0', '0'] |
| 483 | extra_args += ['--glsl-remap-ext-framebuffer-fetch', '1', '1'] |
| 484 | extra_args += ['--glsl-remap-ext-framebuffer-fetch', '2', '2'] |
| 485 | extra_args += ['--glsl-remap-ext-framebuffer-fetch', '3', '3'] |
Hans-Kristian Arntzen | b8905bb | 2020-03-26 11:21:23 +0100 | [diff] [blame] | 486 | if '.zero-initialize.' in shader: |
| 487 | extra_args += ['--force-zero-initialized-variables'] |
Hans-Kristian Arntzen | 41f7e5b | 2017-01-13 16:41:27 +0100 | [diff] [blame] | 488 | |
Hans-Kristian Arntzen | 3a7cb0c | 2019-04-12 14:44:24 +0200 | [diff] [blame] | 489 | spirv_cross_path = paths.spirv_cross |
Hans-Kristian Arntzen | dbee4e4 | 2016-05-05 10:16:22 +0200 | [diff] [blame] | 490 | |
Hans-Kristian Arntzen | 8869a16 | 2016-05-11 19:55:57 +0200 | [diff] [blame] | 491 | # A shader might not be possible to make valid GLSL from, skip validation for this case. |
Hans-Kristian Arntzen | 04e877d | 2020-03-19 14:20:37 +0100 | [diff] [blame] | 492 | if (not ('nocompat' in glsl_path)) or (not vulkan): |
Hans-Kristian Arntzen | 146ea76 | 2018-04-11 10:28:39 +0200 | [diff] [blame] | 493 | subprocess.check_call([spirv_cross_path, '--entry', 'main', '--output', glsl_path, spirv_path] + extra_args) |
Hans-Kristian Arntzen | 04e877d | 2020-03-19 14:20:37 +0100 | [diff] [blame] | 494 | if not 'nocompat' in glsl_path: |
| 495 | validate_shader(glsl_path, False, paths) |
Hans-Kristian Arntzen | 146ea76 | 2018-04-11 10:28:39 +0200 | [diff] [blame] | 496 | else: |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 497 | remove_file(glsl_path) |
Hans-Kristian Arntzen | 146ea76 | 2018-04-11 10:28:39 +0200 | [diff] [blame] | 498 | glsl_path = None |
Hans-Kristian Arntzen | dbee4e4 | 2016-05-05 10:16:22 +0200 | [diff] [blame] | 499 | |
Hans-Kristian Arntzen | bae76d7 | 2020-06-29 12:50:31 +0200 | [diff] [blame] | 500 | if (vulkan or spirv) and (not is_legacy): |
Arseny Kapoulkine | 24c6625 | 2017-01-16 14:19:49 -0800 | [diff] [blame] | 501 | subprocess.check_call([spirv_cross_path, '--entry', 'main', '--vulkan-semantics', '--output', vulkan_glsl_path, spirv_path] + extra_args) |
Hans-Kristian Arntzen | 074f765 | 2019-03-07 12:36:16 +0100 | [diff] [blame] | 502 | validate_shader(vulkan_glsl_path, True, paths) |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 503 | # SPIR-V shaders might just want to validate Vulkan GLSL output, we don't always care about the output. |
| 504 | if not vulkan: |
| 505 | remove_file(vulkan_glsl_path) |
Hans-Kristian Arntzen | dbee4e4 | 2016-05-05 10:16:22 +0200 | [diff] [blame] | 506 | |
| 507 | return (spirv_path, glsl_path, vulkan_glsl_path if vulkan else None) |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 508 | |
Hans-Kristian Arntzen | ad0c5c6 | 2017-02-04 10:19:44 +0100 | [diff] [blame] | 509 | def make_unix_newline(buf): |
| 510 | decoded = codecs.decode(buf, 'utf-8') |
| 511 | decoded = decoded.replace('\r', '') |
| 512 | return codecs.encode(decoded, 'utf-8') |
| 513 | |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 514 | def md5_for_file(path): |
| 515 | md5 = hashlib.md5() |
| 516 | with open(path, 'rb') as f: |
Hans-Kristian Arntzen | ad0c5c6 | 2017-02-04 10:19:44 +0100 | [diff] [blame] | 517 | for chunk in iter(lambda: make_unix_newline(f.read(8192)), b''): |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 518 | md5.update(chunk) |
| 519 | return md5.digest() |
| 520 | |
| 521 | def make_reference_dir(path): |
| 522 | base = os.path.dirname(path) |
| 523 | if not os.path.exists(base): |
| 524 | os.makedirs(base) |
| 525 | |
Hans-Kristian Arntzen | 230a012 | 2017-11-21 09:52:53 +0100 | [diff] [blame] | 526 | def reference_path(directory, relpath, opt): |
Hans-Kristian Arntzen | 78c9a80 | 2016-05-11 19:39:38 +0200 | [diff] [blame] | 527 | split_paths = os.path.split(directory) |
Hans-Kristian Arntzen | 230a012 | 2017-11-21 09:52:53 +0100 | [diff] [blame] | 528 | reference_dir = os.path.join(split_paths[0], 'reference/' + ('opt/' if opt else '')) |
Hans-Kristian Arntzen | 78c9a80 | 2016-05-11 19:39:38 +0200 | [diff] [blame] | 529 | reference_dir = os.path.join(reference_dir, split_paths[1]) |
| 530 | return os.path.join(reference_dir, relpath) |
| 531 | |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 532 | def regression_check_reflect(shader, json_file, args): |
| 533 | reference = reference_path(shader[0], shader[1], args.opt) + '.json' |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 534 | joined_path = os.path.join(shader[0], shader[1]) |
| 535 | print('Reference shader reflection path:', reference) |
| 536 | if os.path.exists(reference): |
Hans-Kristian Arntzen | 309e994 | 2019-10-04 11:04:52 +0200 | [diff] [blame] | 537 | actual = md5_for_file(json_file) |
| 538 | expected = md5_for_file(reference) |
| 539 | if actual != expected: |
Hans-Kristian Arntzen | 9ae91c2 | 2019-04-10 11:33:52 +0200 | [diff] [blame] | 540 | if args.update: |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 541 | print('Generated reflection json has changed for {}!'.format(reference)) |
| 542 | # If we expect changes, update the reference file. |
| 543 | if os.path.exists(reference): |
| 544 | remove_file(reference) |
| 545 | make_reference_dir(reference) |
| 546 | shutil.move(json_file, reference) |
| 547 | else: |
| 548 | print('Generated reflection json in {} does not match reference {}!'.format(json_file, reference)) |
| 549 | with open(json_file, 'r') as f: |
| 550 | print('') |
| 551 | print('Generated:') |
| 552 | print('======================') |
| 553 | print(f.read()) |
| 554 | print('======================') |
| 555 | print('') |
| 556 | |
| 557 | # Otherwise, fail the test. Keep the shader file around so we can inspect. |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 558 | if not args.keep: |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 559 | remove_file(json_file) |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 560 | |
| 561 | raise RuntimeError('Does not match reference') |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 562 | else: |
| 563 | remove_file(json_file) |
| 564 | else: |
| 565 | print('Found new shader {}. Placing generated source code in {}'.format(joined_path, reference)) |
| 566 | make_reference_dir(reference) |
| 567 | shutil.move(json_file, reference) |
Dan Sinclair | e5af412 | 2019-11-05 10:59:41 -0500 | [diff] [blame] | 568 | |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 569 | def regression_check(shader, glsl, args): |
| 570 | reference = reference_path(shader[0], shader[1], args.opt) |
Hans-Kristian Arntzen | 78c9a80 | 2016-05-11 19:39:38 +0200 | [diff] [blame] | 571 | joined_path = os.path.join(shader[0], shader[1]) |
| 572 | print('Reference shader path:', reference) |
| 573 | |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 574 | if os.path.exists(reference): |
| 575 | if md5_for_file(glsl) != md5_for_file(reference): |
Hans-Kristian Arntzen | 9ae91c2 | 2019-04-10 11:33:52 +0200 | [diff] [blame] | 576 | if args.update: |
Bill Hollings | 77f5812 | 2017-08-11 14:54:58 -0400 | [diff] [blame] | 577 | print('Generated source code has changed for {}!'.format(reference)) |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 578 | # If we expect changes, update the reference file. |
| 579 | if os.path.exists(reference): |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 580 | remove_file(reference) |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 581 | make_reference_dir(reference) |
| 582 | shutil.move(glsl, reference) |
| 583 | else: |
Bill Hollings | 77f5812 | 2017-08-11 14:54:58 -0400 | [diff] [blame] | 584 | print('Generated source code in {} does not match reference {}!'.format(glsl, reference)) |
Hans-Kristian Arntzen | 45c797d | 2016-12-16 13:48:30 +0100 | [diff] [blame] | 585 | with open(glsl, 'r') as f: |
| 586 | print('') |
| 587 | print('Generated:') |
| 588 | print('======================') |
| 589 | print(f.read()) |
| 590 | print('======================') |
| 591 | print('') |
| 592 | |
Hans-Kristian Arntzen | e50e04c | 2016-03-22 14:56:50 +0100 | [diff] [blame] | 593 | # Otherwise, fail the test. Keep the shader file around so we can inspect. |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 594 | if not args.keep: |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 595 | remove_file(glsl) |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 596 | raise RuntimeError('Does not match reference') |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 597 | else: |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 598 | remove_file(glsl) |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 599 | else: |
Bill Hollings | 77f5812 | 2017-08-11 14:54:58 -0400 | [diff] [blame] | 600 | print('Found new shader {}. Placing generated source code in {}'.format(joined_path, reference)) |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 601 | make_reference_dir(reference) |
| 602 | shutil.move(glsl, reference) |
| 603 | |
Hans-Kristian Arntzen | 8869a16 | 2016-05-11 19:55:57 +0200 | [diff] [blame] | 604 | def shader_is_vulkan(shader): |
| 605 | return '.vk.' in shader |
| 606 | |
Hans-Kristian Arntzen | 2c7d2e4 | 2016-07-11 12:47:46 +0200 | [diff] [blame] | 607 | def shader_is_desktop(shader): |
| 608 | return '.desktop.' in shader |
| 609 | |
Hans-Kristian Arntzen | f61a5d1 | 2016-08-26 12:58:50 +0200 | [diff] [blame] | 610 | def shader_is_eliminate_dead_variables(shader): |
| 611 | return '.noeliminate.' not in shader |
| 612 | |
Hans-Kristian Arntzen | 45ad58a | 2016-05-10 23:39:41 +0200 | [diff] [blame] | 613 | def shader_is_spirv(shader): |
| 614 | return '.asm.' in shader |
| 615 | |
Hans-Kristian Arntzen | 706d3ea | 2016-09-12 20:11:30 +0200 | [diff] [blame] | 616 | def shader_is_invalid_spirv(shader): |
| 617 | return '.invalid.' in shader |
| 618 | |
Hans-Kristian Arntzen | 41f7e5b | 2017-01-13 16:41:27 +0100 | [diff] [blame] | 619 | def shader_is_legacy(shader): |
| 620 | return '.legacy.' in shader |
| 621 | |
Arseny Kapoulkine | 24c6625 | 2017-01-16 14:19:49 -0800 | [diff] [blame] | 622 | def shader_is_flatten_ubo(shader): |
| 623 | return '.flatten.' in shader |
| 624 | |
Hans-Kristian Arntzen | 36650c8 | 2017-05-22 15:30:43 +0200 | [diff] [blame] | 625 | def shader_is_sso(shader): |
| 626 | return '.sso.' in shader |
| 627 | |
Hans-Kristian Arntzen | 470ae7a | 2017-05-22 17:40:00 +0200 | [diff] [blame] | 628 | def shader_is_flatten_dimensions(shader): |
| 629 | return '.flatten_dim.' in shader |
| 630 | |
Hans-Kristian Arntzen | 7e02f7f | 2017-11-22 11:04:29 +0100 | [diff] [blame] | 631 | def shader_is_noopt(shader): |
| 632 | return '.noopt.' in shader |
| 633 | |
Hans-Kristian Arntzen | 0474848 | 2019-03-19 10:58:37 +0100 | [diff] [blame] | 634 | def shader_is_push_ubo(shader): |
| 635 | return '.push-ubo.' in shader |
| 636 | |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 637 | def test_shader(stats, shader, args, paths): |
Hans-Kristian Arntzen | 78c9a80 | 2016-05-11 19:39:38 +0200 | [diff] [blame] | 638 | joined_path = os.path.join(shader[0], shader[1]) |
Hans-Kristian Arntzen | 8869a16 | 2016-05-11 19:55:57 +0200 | [diff] [blame] | 639 | vulkan = shader_is_vulkan(shader[1]) |
Hans-Kristian Arntzen | 2c7d2e4 | 2016-07-11 12:47:46 +0200 | [diff] [blame] | 640 | desktop = shader_is_desktop(shader[1]) |
Hans-Kristian Arntzen | f61a5d1 | 2016-08-26 12:58:50 +0200 | [diff] [blame] | 641 | eliminate = shader_is_eliminate_dead_variables(shader[1]) |
Hans-Kristian Arntzen | b684716 | 2016-09-10 12:52:23 +0200 | [diff] [blame] | 642 | is_spirv = shader_is_spirv(shader[1]) |
Hans-Kristian Arntzen | 706d3ea | 2016-09-12 20:11:30 +0200 | [diff] [blame] | 643 | invalid_spirv = shader_is_invalid_spirv(shader[1]) |
Hans-Kristian Arntzen | 41f7e5b | 2017-01-13 16:41:27 +0100 | [diff] [blame] | 644 | is_legacy = shader_is_legacy(shader[1]) |
Arseny Kapoulkine | 24c6625 | 2017-01-16 14:19:49 -0800 | [diff] [blame] | 645 | flatten_ubo = shader_is_flatten_ubo(shader[1]) |
Hans-Kristian Arntzen | 36650c8 | 2017-05-22 15:30:43 +0200 | [diff] [blame] | 646 | sso = shader_is_sso(shader[1]) |
Hans-Kristian Arntzen | 470ae7a | 2017-05-22 17:40:00 +0200 | [diff] [blame] | 647 | flatten_dim = shader_is_flatten_dimensions(shader[1]) |
Hans-Kristian Arntzen | 7e02f7f | 2017-11-22 11:04:29 +0100 | [diff] [blame] | 648 | noopt = shader_is_noopt(shader[1]) |
Hans-Kristian Arntzen | 0474848 | 2019-03-19 10:58:37 +0100 | [diff] [blame] | 649 | push_ubo = shader_is_push_ubo(shader[1]) |
Hans-Kristian Arntzen | 78c9a80 | 2016-05-11 19:39:38 +0200 | [diff] [blame] | 650 | |
| 651 | print('Testing shader:', joined_path) |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 652 | spirv, glsl, vulkan_glsl = cross_compile(joined_path, vulkan, is_spirv, invalid_spirv, eliminate, is_legacy, flatten_ubo, sso, flatten_dim, args.opt and (not noopt), push_ubo, args.iterations, paths) |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 653 | |
Hans-Kristian Arntzen | 8869a16 | 2016-05-11 19:55:57 +0200 | [diff] [blame] | 654 | # Only test GLSL stats if we have a shader following GL semantics. |
Hans-Kristian Arntzen | b684716 | 2016-09-10 12:52:23 +0200 | [diff] [blame] | 655 | if stats and (not vulkan) and (not is_spirv) and (not desktop): |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 656 | cross_stats = get_shader_stats(glsl) |
| 657 | |
Hans-Kristian Arntzen | 146ea76 | 2018-04-11 10:28:39 +0200 | [diff] [blame] | 658 | if glsl: |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 659 | regression_check(shader, glsl, args) |
Hans-Kristian Arntzen | dbee4e4 | 2016-05-05 10:16:22 +0200 | [diff] [blame] | 660 | if vulkan_glsl: |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 661 | regression_check((shader[0], shader[1] + '.vk'), vulkan_glsl, args) |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 662 | |
| 663 | remove_file(spirv) |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 664 | |
Hans-Kristian Arntzen | b684716 | 2016-09-10 12:52:23 +0200 | [diff] [blame] | 665 | if stats and (not vulkan) and (not is_spirv) and (not desktop): |
Hans-Kristian Arntzen | 78c9a80 | 2016-05-11 19:39:38 +0200 | [diff] [blame] | 666 | pristine_stats = get_shader_stats(joined_path) |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 667 | |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 668 | a = [] |
Hans-Kristian Arntzen | 78c9a80 | 2016-05-11 19:39:38 +0200 | [diff] [blame] | 669 | a.append(shader[1]) |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 670 | for i in pristine_stats: |
| 671 | a.append(str(i)) |
| 672 | for i in cross_stats: |
| 673 | a.append(str(i)) |
| 674 | print(','.join(a), file = stats) |
| 675 | |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 676 | def test_shader_msl(stats, shader, args, paths): |
Hans-Kristian Arntzen | 1c28ec6 | 2017-01-18 19:05:57 +0100 | [diff] [blame] | 677 | joined_path = os.path.join(shader[0], shader[1]) |
Bill Hollings | be4cb17 | 2017-01-30 22:55:21 -0500 | [diff] [blame] | 678 | print('\nTesting MSL shader:', joined_path) |
Hans-Kristian Arntzen | db5d49f | 2017-09-29 11:07:11 +0200 | [diff] [blame] | 679 | is_spirv = shader_is_spirv(shader[1]) |
Hans-Kristian Arntzen | 7e02f7f | 2017-11-22 11:04:29 +0100 | [diff] [blame] | 680 | noopt = shader_is_noopt(shader[1]) |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 681 | spirv, msl = cross_compile_msl(joined_path, is_spirv, args.opt and (not noopt), args.iterations, paths) |
| 682 | regression_check(shader, msl, args) |
Bill Hollings | a68b327 | 2017-12-26 16:32:45 -0500 | [diff] [blame] | 683 | |
| 684 | # Uncomment the following line to print the temp SPIR-V file path. |
| 685 | # This temp SPIR-V file is not deleted until after the Metal validation step below. |
| 686 | # If Metal validation fails, the temp SPIR-V file can be copied out and |
| 687 | # used as input to an invocation of spirv-cross to debug from Xcode directly. |
| 688 | # To do so, build spriv-cross using `make DEBUG=1`, then run the spriv-cross |
| 689 | # executable from Xcode using args: `--msl --entry main --output msl_path spirv_path`. |
| 690 | # print('SPRIV shader: ' + spirv) |
Hans-Kristian Arntzen | 70743b9 | 2017-02-05 11:04:45 +0100 | [diff] [blame] | 691 | |
Hans-Kristian Arntzen | 33fc2ad | 2019-06-18 10:35:17 +0200 | [diff] [blame] | 692 | shader_is_msl22 = 'msl22' in joined_path |
Hans-Kristian Arntzen | be1e27a | 2019-06-17 16:06:14 +0200 | [diff] [blame] | 693 | skip_validation = shader_is_msl22 and (not args.msl22) |
Hans-Kristian Arntzen | 33fc2ad | 2019-06-18 10:35:17 +0200 | [diff] [blame] | 694 | if '.invalid.' in joined_path: |
| 695 | skip_validation = True |
Hans-Kristian Arntzen | be1e27a | 2019-06-17 16:06:14 +0200 | [diff] [blame] | 696 | |
| 697 | if (not args.force_no_external_validation) and (not skip_validation): |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 698 | validate_shader_msl(shader, args.opt) |
Hans-Kristian Arntzen | 1c28ec6 | 2017-01-18 19:05:57 +0100 | [diff] [blame] | 699 | |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 700 | remove_file(spirv) |
Bill Hollings | a68b327 | 2017-12-26 16:32:45 -0500 | [diff] [blame] | 701 | |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 702 | def test_shader_hlsl(stats, shader, args, paths): |
Robert Konrad | cec9c70 | 2017-01-26 09:45:17 +0100 | [diff] [blame] | 703 | joined_path = os.path.join(shader[0], shader[1]) |
| 704 | print('Testing HLSL shader:', joined_path) |
Hans-Kristian Arntzen | db5d49f | 2017-09-29 11:07:11 +0200 | [diff] [blame] | 705 | is_spirv = shader_is_spirv(shader[1]) |
Hans-Kristian Arntzen | 7e02f7f | 2017-11-22 11:04:29 +0100 | [diff] [blame] | 706 | noopt = shader_is_noopt(shader[1]) |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 707 | spirv, hlsl = cross_compile_hlsl(joined_path, is_spirv, args.opt and (not noopt), args.force_no_external_validation, args.iterations, paths) |
| 708 | regression_check(shader, hlsl, args) |
Hans-Kristian Arntzen | cd442b9 | 2018-04-27 10:31:39 +0200 | [diff] [blame] | 709 | remove_file(spirv) |
Robert Konrad | cec9c70 | 2017-01-26 09:45:17 +0100 | [diff] [blame] | 710 | |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 711 | def test_shader_reflect(stats, shader, args, paths): |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 712 | joined_path = os.path.join(shader[0], shader[1]) |
| 713 | print('Testing shader reflection:', joined_path) |
| 714 | is_spirv = shader_is_spirv(shader[1]) |
| 715 | noopt = shader_is_noopt(shader[1]) |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 716 | spirv, reflect = cross_compile_reflect(joined_path, is_spirv, args.opt and (not noopt), args.iterations, paths) |
| 717 | regression_check_reflect(shader, reflect, args) |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 718 | remove_file(spirv) |
| 719 | |
Hans-Kristian Arntzen | 852e63e | 2019-03-07 12:43:00 +0100 | [diff] [blame] | 720 | def test_shader_file(relpath, stats, args, backend): |
Hans-Kristian Arntzen | 3a7cb0c | 2019-04-12 14:44:24 +0200 | [diff] [blame] | 721 | paths = Paths(args.spirv_cross, args.glslang, args.spirv_as, args.spirv_val, args.spirv_opt) |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 722 | try: |
| 723 | if backend == 'msl': |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 724 | test_shader_msl(stats, (args.folder, relpath), args, paths) |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 725 | elif backend == 'hlsl': |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 726 | test_shader_hlsl(stats, (args.folder, relpath), args, paths) |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 727 | elif backend == 'reflect': |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 728 | test_shader_reflect(stats, (args.folder, relpath), args, paths) |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 729 | else: |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 730 | test_shader(stats, (args.folder, relpath), args, paths) |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 731 | return None |
| 732 | except Exception as e: |
| 733 | return e |
Brad Davis | d0a67ba | 2018-06-19 14:35:25 -0700 | [diff] [blame] | 734 | |
Hans-Kristian Arntzen | 6157bf3 | 2018-10-26 10:53:11 +0200 | [diff] [blame] | 735 | def test_shaders_helper(stats, backend, args): |
Brad Davis | d0a67ba | 2018-06-19 14:35:25 -0700 | [diff] [blame] | 736 | all_files = [] |
| 737 | for root, dirs, files in os.walk(os.path.join(args.folder)): |
Bill Hollings | be4cb17 | 2017-01-30 22:55:21 -0500 | [diff] [blame] | 738 | files = [ f for f in files if not f.startswith(".") ] #ignore system files (esp OSX) |
Hans-Kristian Arntzen | 78c9a80 | 2016-05-11 19:39:38 +0200 | [diff] [blame] | 739 | for i in files: |
| 740 | path = os.path.join(root, i) |
Brad Davis | d0a67ba | 2018-06-19 14:35:25 -0700 | [diff] [blame] | 741 | relpath = os.path.relpath(path, args.folder) |
| 742 | all_files.append(relpath) |
Hans-Kristian Arntzen | 78c9a80 | 2016-05-11 19:39:38 +0200 | [diff] [blame] | 743 | |
Dan Sinclair | e5af412 | 2019-11-05 10:59:41 -0500 | [diff] [blame] | 744 | # The child processes in parallel execution mode don't have the proper state for the global args variable, so |
Brad Davis | d0a67ba | 2018-06-19 14:35:25 -0700 | [diff] [blame] | 745 | # at this point we need to switch to explicit arguments |
| 746 | if args.parallel: |
Hans-Kristian Arntzen | 35a9b79 | 2020-04-27 16:44:14 +0200 | [diff] [blame] | 747 | with multiprocessing.Pool(multiprocessing.cpu_count()) as pool: |
| 748 | results = [] |
| 749 | for f in all_files: |
| 750 | results.append(pool.apply_async(test_shader_file, |
| 751 | args = (f, stats, args, backend))) |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 752 | |
Hans-Kristian Arntzen | 35a9b79 | 2020-04-27 16:44:14 +0200 | [diff] [blame] | 753 | pool.close() |
| 754 | pool.join() |
| 755 | results_completed = [res.get() for res in results] |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 756 | |
Hans-Kristian Arntzen | 35a9b79 | 2020-04-27 16:44:14 +0200 | [diff] [blame] | 757 | for error in results_completed: |
| 758 | if error is not None: |
| 759 | print('Error:', error) |
| 760 | sys.exit(1) |
| 761 | |
Brad Davis | d0a67ba | 2018-06-19 14:35:25 -0700 | [diff] [blame] | 762 | else: |
| 763 | for i in all_files: |
Hans-Kristian Arntzen | 852e63e | 2019-03-07 12:43:00 +0100 | [diff] [blame] | 764 | e = test_shader_file(i, stats, args, backend) |
Hans-Kristian Arntzen | 973b315 | 2019-01-14 11:23:46 +0100 | [diff] [blame] | 765 | if e is not None: |
| 766 | print('Error:', e) |
| 767 | sys.exit(1) |
Brad Davis | d0a67ba | 2018-06-19 14:35:25 -0700 | [diff] [blame] | 768 | |
Hans-Kristian Arntzen | 6157bf3 | 2018-10-26 10:53:11 +0200 | [diff] [blame] | 769 | def test_shaders(backend, args): |
Brad Davis | d0a67ba | 2018-06-19 14:35:25 -0700 | [diff] [blame] | 770 | if args.malisc: |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 771 | with open('stats.csv', 'w') as stats: |
| 772 | print('Shader,OrigRegs,OrigUniRegs,OrigALUShort,OrigLSShort,OrigTEXShort,OrigALULong,OrigLSLong,OrigTEXLong,CrossRegs,CrossUniRegs,CrossALUShort,CrossLSShort,CrossTEXShort,CrossALULong,CrossLSLong,CrossTEXLong', file = stats) |
Hans-Kristian Arntzen | 6157bf3 | 2018-10-26 10:53:11 +0200 | [diff] [blame] | 773 | test_shaders_helper(stats, backend, args) |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 774 | else: |
Hans-Kristian Arntzen | 6157bf3 | 2018-10-26 10:53:11 +0200 | [diff] [blame] | 775 | test_shaders_helper(None, backend, args) |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 776 | |
| 777 | def main(): |
| 778 | parser = argparse.ArgumentParser(description = 'Script for regression testing.') |
| 779 | parser.add_argument('folder', |
| 780 | help = 'Folder containing shader files to test.') |
| 781 | parser.add_argument('--update', |
| 782 | action = 'store_true', |
| 783 | help = 'Updates reference files if there is a mismatch. Use when legitimate changes in output is found.') |
Hans-Kristian Arntzen | e50e04c | 2016-03-22 14:56:50 +0100 | [diff] [blame] | 784 | parser.add_argument('--keep', |
| 785 | action = 'store_true', |
| 786 | help = 'Leave failed GLSL shaders on disk if they fail regression. Useful for debugging.') |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 787 | parser.add_argument('--malisc', |
| 788 | action = 'store_true', |
Hans-Kristian Arntzen | 147e53a | 2016-04-04 09:36:04 +0200 | [diff] [blame] | 789 | help = 'Use malisc offline compiler to determine static cycle counts before and after spirv-cross.') |
Bill Hollings | 5ad73f3 | 2017-03-19 21:06:21 -0400 | [diff] [blame] | 790 | parser.add_argument('--msl', |
Hans-Kristian Arntzen | 1c28ec6 | 2017-01-18 19:05:57 +0100 | [diff] [blame] | 791 | action = 'store_true', |
| 792 | help = 'Test Metal backend.') |
Bill Hollings | 6fdb534 | 2017-03-19 21:46:06 -0400 | [diff] [blame] | 793 | parser.add_argument('--metal', |
| 794 | action = 'store_true', |
| 795 | help = 'Deprecated Metal option. Use --msl instead.') |
Robert Konrad | cec9c70 | 2017-01-26 09:45:17 +0100 | [diff] [blame] | 796 | parser.add_argument('--hlsl', |
| 797 | action = 'store_true', |
| 798 | help = 'Test HLSL backend.') |
Hans-Kristian Arntzen | 70743b9 | 2017-02-05 11:04:45 +0100 | [diff] [blame] | 799 | parser.add_argument('--force-no-external-validation', |
| 800 | action = 'store_true', |
| 801 | help = 'Disable all external validation.') |
Hans-Kristian Arntzen | 230a012 | 2017-11-21 09:52:53 +0100 | [diff] [blame] | 802 | parser.add_argument('--opt', |
| 803 | action = 'store_true', |
| 804 | help = 'Run SPIRV-Tools optimization passes as well.') |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 805 | parser.add_argument('--reflect', |
| 806 | action = 'store_true', |
| 807 | help = 'Test reflection backend.') |
Brad Davis | d0a67ba | 2018-06-19 14:35:25 -0700 | [diff] [blame] | 808 | parser.add_argument('--parallel', |
| 809 | action = 'store_true', |
| 810 | help = 'Execute tests in parallel. Useful for doing regression quickly, but bad for debugging and stat output.') |
Hans-Kristian Arntzen | 3a7cb0c | 2019-04-12 14:44:24 +0200 | [diff] [blame] | 811 | parser.add_argument('--spirv-cross', |
| 812 | default = './spirv-cross', |
| 813 | help = 'Explicit path to spirv-cross') |
Hans-Kristian Arntzen | 074f765 | 2019-03-07 12:36:16 +0100 | [diff] [blame] | 814 | parser.add_argument('--glslang', |
| 815 | default = 'glslangValidator', |
| 816 | help = 'Explicit path to glslangValidator') |
| 817 | parser.add_argument('--spirv-as', |
| 818 | default = 'spirv-as', |
| 819 | help = 'Explicit path to spirv-as') |
| 820 | parser.add_argument('--spirv-val', |
| 821 | default = 'spirv-val', |
| 822 | help = 'Explicit path to spirv-val') |
| 823 | parser.add_argument('--spirv-opt', |
| 824 | default = 'spirv-opt', |
| 825 | help = 'Explicit path to spirv-opt') |
Hans-Kristian Arntzen | c60b9a1 | 2019-04-08 11:53:13 +0200 | [diff] [blame] | 826 | parser.add_argument('--iterations', |
| 827 | default = 1, |
| 828 | type = int, |
| 829 | help = 'Number of iterations to run SPIRV-Cross (benchmarking)') |
Dan Sinclair | e5af412 | 2019-11-05 10:59:41 -0500 | [diff] [blame] | 830 | |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 831 | args = parser.parse_args() |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 832 | if not args.folder: |
| 833 | sys.stderr.write('Need shader folder.\n') |
| 834 | sys.exit(1) |
| 835 | |
Brad Davis | 7620400 | 2018-06-20 10:25:38 -0700 | [diff] [blame] | 836 | if (args.parallel and (args.malisc or args.force_no_external_validation or args.update)): |
| 837 | sys.stderr.write('Parallel execution is disabled when using the flags --update, --malisc or --force-no-external-validation\n') |
Brad Davis | d0a67ba | 2018-06-19 14:35:25 -0700 | [diff] [blame] | 838 | args.parallel = False |
Dan Sinclair | e5af412 | 2019-11-05 10:59:41 -0500 | [diff] [blame] | 839 | |
Hans-Kristian Arntzen | 856a8d2 | 2019-06-18 12:22:46 +0200 | [diff] [blame] | 840 | args.msl22 = False |
Bill Hollings | 5ad73f3 | 2017-03-19 21:06:21 -0400 | [diff] [blame] | 841 | if args.msl: |
Bill Hollings | be4cb17 | 2017-01-30 22:55:21 -0500 | [diff] [blame] | 842 | print_msl_compiler_version() |
Hans-Kristian Arntzen | be1e27a | 2019-06-17 16:06:14 +0200 | [diff] [blame] | 843 | args.msl22 = msl_compiler_supports_22() |
Arseny Kapoulkine | 49baf41 | 2017-01-25 00:01:53 -0800 | [diff] [blame] | 844 | |
Hans-Kristian Arntzen | 6157bf3 | 2018-10-26 10:53:11 +0200 | [diff] [blame] | 845 | backend = 'glsl' |
Dan Sinclair | e5af412 | 2019-11-05 10:59:41 -0500 | [diff] [blame] | 846 | if (args.msl or args.metal): |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 847 | backend = 'msl' |
Dan Sinclair | e5af412 | 2019-11-05 10:59:41 -0500 | [diff] [blame] | 848 | elif args.hlsl: |
Brad Davis | 709d3c6 | 2018-06-03 11:16:37 -0700 | [diff] [blame] | 849 | backend = 'hlsl' |
| 850 | elif args.reflect: |
| 851 | backend = 'reflect' |
Hans-Kristian Arntzen | 70743b9 | 2017-02-05 11:04:45 +0100 | [diff] [blame] | 852 | |
Hans-Kristian Arntzen | 6157bf3 | 2018-10-26 10:53:11 +0200 | [diff] [blame] | 853 | test_shaders(backend, args) |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 854 | if args.malisc: |
| 855 | print('Stats in stats.csv!') |
Hans-Kristian Arntzen | e50e04c | 2016-03-22 14:56:50 +0100 | [diff] [blame] | 856 | print('Tests completed!') |
Hans-Kristian Arntzen | 75471fb | 2016-03-02 18:09:16 +0100 | [diff] [blame] | 857 | |
| 858 | if __name__ == '__main__': |
Hans-Kristian Arntzen | 0a5b3a6 | 2016-03-22 14:44:12 +0100 | [diff] [blame] | 859 | main() |