blob: f33498279b4779a27d20e27ec78584fa9275d42d [file] [log] [blame]
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +01001#!/usr/bin/env python3
2
3import sys
4import os
Arseny Kapoulkinef45075b2017-01-24 22:26:39 -08005import os.path
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +01006import subprocess
7import tempfile
8import re
9import itertools
10import hashlib
11import shutil
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +010012import argparse
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +010013
Arseny Kapoulkine49baf412017-01-25 00:01:53 -080014METALC = '/Applications/Xcode.app/Contents/Developer/Platforms/iPhoneOS.platform/usr/bin/metal'
15
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +010016def parse_stats(stats):
17 m = re.search('([0-9]+) work registers', stats)
18 registers = int(m.group(1)) if m else 0
19
20 m = re.search('([0-9]+) uniform registers', stats)
21 uniform_regs = int(m.group(1)) if m else 0
22
23 m_list = re.findall('(-?[0-9]+)\s+(-?[0-9]+)\s+(-?[0-9]+)', stats)
24 alu_short = float(m_list[1][0]) if m_list else 0
25 ls_short = float(m_list[1][1]) if m_list else 0
26 tex_short = float(m_list[1][2]) if m_list else 0
27 alu_long = float(m_list[2][0]) if m_list else 0
28 ls_long = float(m_list[2][1]) if m_list else 0
29 tex_long = float(m_list[2][2]) if m_list else 0
30
31 return (registers, uniform_regs, alu_short, ls_short, tex_short, alu_long, ls_long, tex_long)
32
33def get_shader_type(shader):
34 _, ext = os.path.splitext(shader)
35 if ext == '.vert':
36 return '--vertex'
37 elif ext == '.frag':
38 return '--fragment'
39 elif ext == '.comp':
40 return '--compute'
41 elif ext == '.tesc':
42 return '--tessellation_control'
43 elif ext == '.tese':
44 return '--tessellation_evaluation'
45 elif ext == '.geom':
46 return '--geometry'
47 else:
48 return ''
49
50def get_shader_stats(shader):
51 f, path = tempfile.mkstemp()
52
53 os.close(f)
54 p = subprocess.Popen(['malisc', get_shader_type(shader), '--core', 'Mali-T760', '-V', shader], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
55 stdout, stderr = p.communicate()
56 os.remove(path)
57
58 if p.returncode != 0:
59 print(stderr.decode('utf-8'))
60 raise OSError('malisc failed')
61 p.wait()
62
63 returned = stdout.decode('utf-8')
64 return parse_stats(returned)
65
Arseny Kapoulkinef45075b2017-01-24 22:26:39 -080066def validate_shader_msl(shader):
Arseny Kapoulkine49baf412017-01-25 00:01:53 -080067 subprocess.check_call([METALC, '-x', 'metal', '-std=ios-metal1.0', '-Werror', shader])
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +010068
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +010069def cross_compile_msl(shader):
70 spirv_f, spirv_path = tempfile.mkstemp()
71 msl_f, msl_path = tempfile.mkstemp(suffix = os.path.basename(shader))
72 os.close(spirv_f)
73 os.close(msl_f)
74 subprocess.check_call(['glslangValidator', '-V', '-o', spirv_path, shader])
75 spirv_cross_path = './spirv-cross'
76 subprocess.check_call([spirv_cross_path, '--entry', 'main', '--output', msl_path, spirv_path, '--metal'])
77 subprocess.check_call(['spirv-val', spirv_path])
78
Arseny Kapoulkine49baf412017-01-25 00:01:53 -080079 if os.path.exists(METALC):
80 validate_shader_msl(msl_path)
Arseny Kapoulkinef45075b2017-01-24 22:26:39 -080081
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +010082 return (spirv_path, msl_path)
83
Arseny Kapoulkinef45075b2017-01-24 22:26:39 -080084def validate_shader(shader, vulkan):
85 if vulkan:
86 subprocess.check_call(['glslangValidator', '-V', shader])
87 else:
88 subprocess.check_call(['glslangValidator', shader])
89
Arseny Kapoulkine24c66252017-01-16 14:19:49 -080090def cross_compile(shader, vulkan, spirv, invalid_spirv, eliminate, is_legacy, flatten_ubo):
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +010091 spirv_f, spirv_path = tempfile.mkstemp()
92 glsl_f, glsl_path = tempfile.mkstemp(suffix = os.path.basename(shader))
93 os.close(spirv_f)
94 os.close(glsl_f)
95
Hans-Kristian Arntzen45ad58a2016-05-10 23:39:41 +020096 if vulkan or spirv:
Hans-Kristian Arntzendbee4e42016-05-05 10:16:22 +020097 vulkan_glsl_f, vulkan_glsl_path = tempfile.mkstemp(suffix = os.path.basename(shader))
98 os.close(vulkan_glsl_f)
99
Hans-Kristian Arntzen45ad58a2016-05-10 23:39:41 +0200100 if spirv:
101 subprocess.check_call(['spirv-as', '-o', spirv_path, shader])
102 else:
Hans-Kristian Arntzenb6847162016-09-10 12:52:23 +0200103 subprocess.check_call(['glslangValidator', '-V', '-o', spirv_path, shader])
Hans-Kristian Arntzen45ad58a2016-05-10 23:39:41 +0200104
Hans-Kristian Arntzen706d3ea2016-09-12 20:11:30 +0200105 if not invalid_spirv:
106 subprocess.check_call(['spirv-val', spirv_path])
Hans-Kristian Arntzendbee4e42016-05-05 10:16:22 +0200107
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800108 extra_args = []
109 if eliminate:
110 extra_args += ['--remove-unused-variables']
Hans-Kristian Arntzen41f7e5b2017-01-13 16:41:27 +0100111 if is_legacy:
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800112 extra_args += ['--version', '100', '--es']
113 if flatten_ubo:
114 extra_args += ['--flatten-ubo']
Hans-Kristian Arntzen41f7e5b2017-01-13 16:41:27 +0100115
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200116 spirv_cross_path = './spirv-cross'
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800117 subprocess.check_call([spirv_cross_path, '--entry', 'main', '--output', glsl_path, spirv_path] + extra_args)
Hans-Kristian Arntzendbee4e42016-05-05 10:16:22 +0200118
Hans-Kristian Arntzen8869a162016-05-11 19:55:57 +0200119 # A shader might not be possible to make valid GLSL from, skip validation for this case.
Hans-Kristian Arntzen45ad58a2016-05-10 23:39:41 +0200120 if (not ('nocompat' in glsl_path)) and (not spirv):
Hans-Kristian Arntzendbee4e42016-05-05 10:16:22 +0200121 validate_shader(glsl_path, False)
122
Hans-Kristian Arntzen45ad58a2016-05-10 23:39:41 +0200123 if vulkan or spirv:
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800124 subprocess.check_call([spirv_cross_path, '--entry', 'main', '--vulkan-semantics', '--output', vulkan_glsl_path, spirv_path] + extra_args)
Hans-Kristian Arntzendbee4e42016-05-05 10:16:22 +0200125 validate_shader(vulkan_glsl_path, vulkan)
126
127 return (spirv_path, glsl_path, vulkan_glsl_path if vulkan else None)
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100128
129def md5_for_file(path):
130 md5 = hashlib.md5()
131 with open(path, 'rb') as f:
132 for chunk in iter(lambda: f.read(8192), b''):
133 md5.update(chunk)
134 return md5.digest()
135
136def make_reference_dir(path):
137 base = os.path.dirname(path)
138 if not os.path.exists(base):
139 os.makedirs(base)
140
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200141def reference_path(directory, relpath):
142 split_paths = os.path.split(directory)
143 reference_dir = os.path.join(split_paths[0], 'reference/')
144 reference_dir = os.path.join(reference_dir, split_paths[1])
145 return os.path.join(reference_dir, relpath)
146
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100147def regression_check(shader, glsl, update, keep):
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200148 reference = reference_path(shader[0], shader[1])
149 joined_path = os.path.join(shader[0], shader[1])
150 print('Reference shader path:', reference)
151
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100152 if os.path.exists(reference):
153 if md5_for_file(glsl) != md5_for_file(reference):
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100154 if update:
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100155 print('Generated GLSL has changed for {}!'.format(reference))
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100156 # If we expect changes, update the reference file.
157 if os.path.exists(reference):
158 os.remove(reference)
159 make_reference_dir(reference)
160 shutil.move(glsl, reference)
161 else:
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100162 print('Generated GLSL in {} does not match reference {}!'.format(glsl, reference))
Hans-Kristian Arntzen45c797d2016-12-16 13:48:30 +0100163 with open(glsl, 'r') as f:
164 print('')
165 print('Generated:')
166 print('======================')
167 print(f.read())
168 print('======================')
169 print('')
170
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100171 # Otherwise, fail the test. Keep the shader file around so we can inspect.
172 if not keep:
173 os.remove(glsl)
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100174 sys.exit(1)
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100175 else:
176 os.remove(glsl)
177 else:
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200178 print('Found new shader {}. Placing GLSL in {}'.format(joined_path, reference))
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100179 make_reference_dir(reference)
180 shutil.move(glsl, reference)
181
Hans-Kristian Arntzen8869a162016-05-11 19:55:57 +0200182def shader_is_vulkan(shader):
183 return '.vk.' in shader
184
Hans-Kristian Arntzen2c7d2e42016-07-11 12:47:46 +0200185def shader_is_desktop(shader):
186 return '.desktop.' in shader
187
Hans-Kristian Arntzenf61a5d12016-08-26 12:58:50 +0200188def shader_is_eliminate_dead_variables(shader):
189 return '.noeliminate.' not in shader
190
Hans-Kristian Arntzen45ad58a2016-05-10 23:39:41 +0200191def shader_is_spirv(shader):
192 return '.asm.' in shader
193
Hans-Kristian Arntzen706d3ea2016-09-12 20:11:30 +0200194def shader_is_invalid_spirv(shader):
195 return '.invalid.' in shader
196
Hans-Kristian Arntzen41f7e5b2017-01-13 16:41:27 +0100197def shader_is_legacy(shader):
198 return '.legacy.' in shader
199
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800200def shader_is_flatten_ubo(shader):
201 return '.flatten.' in shader
202
Hans-Kristian Arntzen8869a162016-05-11 19:55:57 +0200203def test_shader(stats, shader, update, keep):
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200204 joined_path = os.path.join(shader[0], shader[1])
Hans-Kristian Arntzen8869a162016-05-11 19:55:57 +0200205 vulkan = shader_is_vulkan(shader[1])
Hans-Kristian Arntzen2c7d2e42016-07-11 12:47:46 +0200206 desktop = shader_is_desktop(shader[1])
Hans-Kristian Arntzenf61a5d12016-08-26 12:58:50 +0200207 eliminate = shader_is_eliminate_dead_variables(shader[1])
Hans-Kristian Arntzenb6847162016-09-10 12:52:23 +0200208 is_spirv = shader_is_spirv(shader[1])
Hans-Kristian Arntzen706d3ea2016-09-12 20:11:30 +0200209 invalid_spirv = shader_is_invalid_spirv(shader[1])
Hans-Kristian Arntzen41f7e5b2017-01-13 16:41:27 +0100210 is_legacy = shader_is_legacy(shader[1])
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800211 flatten_ubo = shader_is_flatten_ubo(shader[1])
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200212
213 print('Testing shader:', joined_path)
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800214 spirv, glsl, vulkan_glsl = cross_compile(joined_path, vulkan, is_spirv, invalid_spirv, eliminate, is_legacy, flatten_ubo)
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100215
Hans-Kristian Arntzen8869a162016-05-11 19:55:57 +0200216 # Only test GLSL stats if we have a shader following GL semantics.
Hans-Kristian Arntzenb6847162016-09-10 12:52:23 +0200217 if stats and (not vulkan) and (not is_spirv) and (not desktop):
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100218 cross_stats = get_shader_stats(glsl)
219
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100220 regression_check(shader, glsl, update, keep)
Hans-Kristian Arntzendbee4e42016-05-05 10:16:22 +0200221 if vulkan_glsl:
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200222 regression_check((shader[0], shader[1] + '.vk'), vulkan_glsl, update, keep)
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100223 os.remove(spirv)
224
Hans-Kristian Arntzenb6847162016-09-10 12:52:23 +0200225 if stats and (not vulkan) and (not is_spirv) and (not desktop):
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200226 pristine_stats = get_shader_stats(joined_path)
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100227
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100228 a = []
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200229 a.append(shader[1])
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100230 for i in pristine_stats:
231 a.append(str(i))
232 for i in cross_stats:
233 a.append(str(i))
234 print(','.join(a), file = stats)
235
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100236def test_shader_msl(stats, shader, update, keep):
237 joined_path = os.path.join(shader[0], shader[1])
238 print('Testing MSL shader:', joined_path)
239 spirv, msl = cross_compile_msl(joined_path)
240 regression_check(shader, msl, update, keep)
241 os.remove(spirv)
242
243def test_shaders_helper(stats, shader_dir, update, malisc, keep, backend):
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200244 for root, dirs, files in os.walk(os.path.join(shader_dir)):
245 for i in files:
246 path = os.path.join(root, i)
247 relpath = os.path.relpath(path, shader_dir)
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100248 if backend == 'metal':
249 test_shader_msl(stats, (shader_dir, relpath), update, keep)
250 else:
251 test_shader(stats, (shader_dir, relpath), update, keep)
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200252
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100253def test_shaders(shader_dir, update, malisc, keep, backend):
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100254 if malisc:
255 with open('stats.csv', 'w') as stats:
256 print('Shader,OrigRegs,OrigUniRegs,OrigALUShort,OrigLSShort,OrigTEXShort,OrigALULong,OrigLSLong,OrigTEXLong,CrossRegs,CrossUniRegs,CrossALUShort,CrossLSShort,CrossTEXShort,CrossALULong,CrossLSLong,CrossTEXLong', file = stats)
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100257 test_shaders_helper(stats, shader_dir, update, malisc, keep, backend)
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100258 else:
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100259 test_shaders_helper(None, shader_dir, update, malisc, keep, backend)
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100260
261def main():
262 parser = argparse.ArgumentParser(description = 'Script for regression testing.')
263 parser.add_argument('folder',
264 help = 'Folder containing shader files to test.')
265 parser.add_argument('--update',
266 action = 'store_true',
267 help = 'Updates reference files if there is a mismatch. Use when legitimate changes in output is found.')
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100268 parser.add_argument('--keep',
269 action = 'store_true',
270 help = 'Leave failed GLSL shaders on disk if they fail regression. Useful for debugging.')
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100271 parser.add_argument('--malisc',
272 action = 'store_true',
Hans-Kristian Arntzen147e53a2016-04-04 09:36:04 +0200273 help = 'Use malisc offline compiler to determine static cycle counts before and after spirv-cross.')
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100274 parser.add_argument('--metal',
275 action = 'store_true',
276 help = 'Test Metal backend.')
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100277 args = parser.parse_args()
278
279 if not args.folder:
280 sys.stderr.write('Need shader folder.\n')
281 sys.exit(1)
282
Arseny Kapoulkine49baf412017-01-25 00:01:53 -0800283 if os.path.exists(METALC):
284 subprocess.check_call([METALC, '--version'])
285
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100286 test_shaders(args.folder, args.update, args.malisc, args.keep, 'metal' if args.metal else 'glsl')
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100287 if args.malisc:
288 print('Stats in stats.csv!')
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100289 print('Tests completed!')
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100290
291if __name__ == '__main__':
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100292 main()