blob: 1ad768e935c16dfdb109cecf593190d1aaa7019f [file] [log] [blame]
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +01001#!/usr/bin/env python3
2
3import sys
4import os
5import subprocess
6import tempfile
7import re
8import itertools
9import hashlib
10import shutil
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +010011import argparse
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +010012
13def parse_stats(stats):
14 m = re.search('([0-9]+) work registers', stats)
15 registers = int(m.group(1)) if m else 0
16
17 m = re.search('([0-9]+) uniform registers', stats)
18 uniform_regs = int(m.group(1)) if m else 0
19
20 m_list = re.findall('(-?[0-9]+)\s+(-?[0-9]+)\s+(-?[0-9]+)', stats)
21 alu_short = float(m_list[1][0]) if m_list else 0
22 ls_short = float(m_list[1][1]) if m_list else 0
23 tex_short = float(m_list[1][2]) if m_list else 0
24 alu_long = float(m_list[2][0]) if m_list else 0
25 ls_long = float(m_list[2][1]) if m_list else 0
26 tex_long = float(m_list[2][2]) if m_list else 0
27
28 return (registers, uniform_regs, alu_short, ls_short, tex_short, alu_long, ls_long, tex_long)
29
30def get_shader_type(shader):
31 _, ext = os.path.splitext(shader)
32 if ext == '.vert':
33 return '--vertex'
34 elif ext == '.frag':
35 return '--fragment'
36 elif ext == '.comp':
37 return '--compute'
38 elif ext == '.tesc':
39 return '--tessellation_control'
40 elif ext == '.tese':
41 return '--tessellation_evaluation'
42 elif ext == '.geom':
43 return '--geometry'
44 else:
45 return ''
46
47def get_shader_stats(shader):
48 f, path = tempfile.mkstemp()
49
50 os.close(f)
51 p = subprocess.Popen(['malisc', get_shader_type(shader), '--core', 'Mali-T760', '-V', shader], stdout = subprocess.PIPE, stderr = subprocess.PIPE)
52 stdout, stderr = p.communicate()
53 os.remove(path)
54
55 if p.returncode != 0:
56 print(stderr.decode('utf-8'))
57 raise OSError('malisc failed')
58 p.wait()
59
60 returned = stdout.decode('utf-8')
61 return parse_stats(returned)
62
Hans-Kristian Arntzen7652c902016-04-19 11:13:47 +020063def validate_shader(shader, vulkan):
64 if vulkan:
65 subprocess.check_call(['glslangValidator', '-V', shader])
66 else:
67 subprocess.check_call(['glslangValidator', shader])
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +010068
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +010069def cross_compile_msl(shader):
70 spirv_f, spirv_path = tempfile.mkstemp()
71 msl_f, msl_path = tempfile.mkstemp(suffix = os.path.basename(shader))
72 os.close(spirv_f)
73 os.close(msl_f)
74 subprocess.check_call(['glslangValidator', '-V', '-o', spirv_path, shader])
75 spirv_cross_path = './spirv-cross'
76 subprocess.check_call([spirv_cross_path, '--entry', 'main', '--output', msl_path, spirv_path, '--metal'])
77 subprocess.check_call(['spirv-val', spirv_path])
78
79 # TODO: Add optional validation of the MSL output.
80 return (spirv_path, msl_path)
81
Robert Konradcec9c702017-01-26 09:45:17 +010082def cross_compile_hlsl(shader):
83 spirv_f, spirv_path = tempfile.mkstemp()
84 hlsl_f, hlsl_path = tempfile.mkstemp(suffix = os.path.basename(shader))
85 os.close(spirv_f)
86 os.close(hlsl_f)
87 subprocess.check_call(['glslangValidator', '-V', '-o', spirv_path, shader])
88 spirv_cross_path = './spirv-cross'
89 subprocess.check_call([spirv_cross_path, '--entry', 'main', '--output', msl_path, spirv_path, '--hlsl'])
90 subprocess.check_call(['spirv-val', spirv_path])
91
92 # TODO: Add optional validation of the HLSL output.
93 return (spirv_path, msl_path)
94
Arseny Kapoulkine24c66252017-01-16 14:19:49 -080095def cross_compile(shader, vulkan, spirv, invalid_spirv, eliminate, is_legacy, flatten_ubo):
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +010096 spirv_f, spirv_path = tempfile.mkstemp()
97 glsl_f, glsl_path = tempfile.mkstemp(suffix = os.path.basename(shader))
98 os.close(spirv_f)
99 os.close(glsl_f)
100
Hans-Kristian Arntzen45ad58a2016-05-10 23:39:41 +0200101 if vulkan or spirv:
Hans-Kristian Arntzendbee4e42016-05-05 10:16:22 +0200102 vulkan_glsl_f, vulkan_glsl_path = tempfile.mkstemp(suffix = os.path.basename(shader))
103 os.close(vulkan_glsl_f)
104
Hans-Kristian Arntzen45ad58a2016-05-10 23:39:41 +0200105 if spirv:
106 subprocess.check_call(['spirv-as', '-o', spirv_path, shader])
107 else:
Hans-Kristian Arntzenb6847162016-09-10 12:52:23 +0200108 subprocess.check_call(['glslangValidator', '-V', '-o', spirv_path, shader])
Hans-Kristian Arntzen45ad58a2016-05-10 23:39:41 +0200109
Hans-Kristian Arntzen706d3ea2016-09-12 20:11:30 +0200110 if not invalid_spirv:
111 subprocess.check_call(['spirv-val', spirv_path])
Hans-Kristian Arntzendbee4e42016-05-05 10:16:22 +0200112
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800113 extra_args = []
114 if eliminate:
115 extra_args += ['--remove-unused-variables']
Hans-Kristian Arntzen41f7e5b2017-01-13 16:41:27 +0100116 if is_legacy:
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800117 extra_args += ['--version', '100', '--es']
118 if flatten_ubo:
119 extra_args += ['--flatten-ubo']
Hans-Kristian Arntzen41f7e5b2017-01-13 16:41:27 +0100120
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200121 spirv_cross_path = './spirv-cross'
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800122 subprocess.check_call([spirv_cross_path, '--entry', 'main', '--output', glsl_path, spirv_path] + extra_args)
Hans-Kristian Arntzendbee4e42016-05-05 10:16:22 +0200123
Hans-Kristian Arntzen8869a162016-05-11 19:55:57 +0200124 # A shader might not be possible to make valid GLSL from, skip validation for this case.
Hans-Kristian Arntzen45ad58a2016-05-10 23:39:41 +0200125 if (not ('nocompat' in glsl_path)) and (not spirv):
Hans-Kristian Arntzendbee4e42016-05-05 10:16:22 +0200126 validate_shader(glsl_path, False)
127
Hans-Kristian Arntzen45ad58a2016-05-10 23:39:41 +0200128 if vulkan or spirv:
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800129 subprocess.check_call([spirv_cross_path, '--entry', 'main', '--vulkan-semantics', '--output', vulkan_glsl_path, spirv_path] + extra_args)
Hans-Kristian Arntzendbee4e42016-05-05 10:16:22 +0200130 validate_shader(vulkan_glsl_path, vulkan)
131
132 return (spirv_path, glsl_path, vulkan_glsl_path if vulkan else None)
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100133
134def md5_for_file(path):
135 md5 = hashlib.md5()
136 with open(path, 'rb') as f:
137 for chunk in iter(lambda: f.read(8192), b''):
138 md5.update(chunk)
139 return md5.digest()
140
141def make_reference_dir(path):
142 base = os.path.dirname(path)
143 if not os.path.exists(base):
144 os.makedirs(base)
145
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200146def reference_path(directory, relpath):
147 split_paths = os.path.split(directory)
148 reference_dir = os.path.join(split_paths[0], 'reference/')
149 reference_dir = os.path.join(reference_dir, split_paths[1])
150 return os.path.join(reference_dir, relpath)
151
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100152def regression_check(shader, glsl, update, keep):
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200153 reference = reference_path(shader[0], shader[1])
154 joined_path = os.path.join(shader[0], shader[1])
155 print('Reference shader path:', reference)
156
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100157 if os.path.exists(reference):
158 if md5_for_file(glsl) != md5_for_file(reference):
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100159 if update:
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100160 print('Generated GLSL has changed for {}!'.format(reference))
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100161 # If we expect changes, update the reference file.
162 if os.path.exists(reference):
163 os.remove(reference)
164 make_reference_dir(reference)
165 shutil.move(glsl, reference)
166 else:
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100167 print('Generated GLSL in {} does not match reference {}!'.format(glsl, reference))
Hans-Kristian Arntzen45c797d2016-12-16 13:48:30 +0100168 with open(glsl, 'r') as f:
169 print('')
170 print('Generated:')
171 print('======================')
172 print(f.read())
173 print('======================')
174 print('')
175
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100176 # Otherwise, fail the test. Keep the shader file around so we can inspect.
177 if not keep:
178 os.remove(glsl)
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100179 sys.exit(1)
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100180 else:
181 os.remove(glsl)
182 else:
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200183 print('Found new shader {}. Placing GLSL in {}'.format(joined_path, reference))
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100184 make_reference_dir(reference)
185 shutil.move(glsl, reference)
186
Hans-Kristian Arntzen8869a162016-05-11 19:55:57 +0200187def shader_is_vulkan(shader):
188 return '.vk.' in shader
189
Hans-Kristian Arntzen2c7d2e42016-07-11 12:47:46 +0200190def shader_is_desktop(shader):
191 return '.desktop.' in shader
192
Hans-Kristian Arntzenf61a5d12016-08-26 12:58:50 +0200193def shader_is_eliminate_dead_variables(shader):
194 return '.noeliminate.' not in shader
195
Hans-Kristian Arntzen45ad58a2016-05-10 23:39:41 +0200196def shader_is_spirv(shader):
197 return '.asm.' in shader
198
Hans-Kristian Arntzen706d3ea2016-09-12 20:11:30 +0200199def shader_is_invalid_spirv(shader):
200 return '.invalid.' in shader
201
Hans-Kristian Arntzen41f7e5b2017-01-13 16:41:27 +0100202def shader_is_legacy(shader):
203 return '.legacy.' in shader
204
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800205def shader_is_flatten_ubo(shader):
206 return '.flatten.' in shader
207
Hans-Kristian Arntzen8869a162016-05-11 19:55:57 +0200208def test_shader(stats, shader, update, keep):
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200209 joined_path = os.path.join(shader[0], shader[1])
Hans-Kristian Arntzen8869a162016-05-11 19:55:57 +0200210 vulkan = shader_is_vulkan(shader[1])
Hans-Kristian Arntzen2c7d2e42016-07-11 12:47:46 +0200211 desktop = shader_is_desktop(shader[1])
Hans-Kristian Arntzenf61a5d12016-08-26 12:58:50 +0200212 eliminate = shader_is_eliminate_dead_variables(shader[1])
Hans-Kristian Arntzenb6847162016-09-10 12:52:23 +0200213 is_spirv = shader_is_spirv(shader[1])
Hans-Kristian Arntzen706d3ea2016-09-12 20:11:30 +0200214 invalid_spirv = shader_is_invalid_spirv(shader[1])
Hans-Kristian Arntzen41f7e5b2017-01-13 16:41:27 +0100215 is_legacy = shader_is_legacy(shader[1])
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800216 flatten_ubo = shader_is_flatten_ubo(shader[1])
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200217
218 print('Testing shader:', joined_path)
Arseny Kapoulkine24c66252017-01-16 14:19:49 -0800219 spirv, glsl, vulkan_glsl = cross_compile(joined_path, vulkan, is_spirv, invalid_spirv, eliminate, is_legacy, flatten_ubo)
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100220
Hans-Kristian Arntzen8869a162016-05-11 19:55:57 +0200221 # Only test GLSL stats if we have a shader following GL semantics.
Hans-Kristian Arntzenb6847162016-09-10 12:52:23 +0200222 if stats and (not vulkan) and (not is_spirv) and (not desktop):
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100223 cross_stats = get_shader_stats(glsl)
224
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100225 regression_check(shader, glsl, update, keep)
Hans-Kristian Arntzendbee4e42016-05-05 10:16:22 +0200226 if vulkan_glsl:
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200227 regression_check((shader[0], shader[1] + '.vk'), vulkan_glsl, update, keep)
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100228 os.remove(spirv)
229
Hans-Kristian Arntzenb6847162016-09-10 12:52:23 +0200230 if stats and (not vulkan) and (not is_spirv) and (not desktop):
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200231 pristine_stats = get_shader_stats(joined_path)
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100232
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100233 a = []
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200234 a.append(shader[1])
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100235 for i in pristine_stats:
236 a.append(str(i))
237 for i in cross_stats:
238 a.append(str(i))
239 print(','.join(a), file = stats)
240
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100241def test_shader_msl(stats, shader, update, keep):
242 joined_path = os.path.join(shader[0], shader[1])
243 print('Testing MSL shader:', joined_path)
244 spirv, msl = cross_compile_msl(joined_path)
245 regression_check(shader, msl, update, keep)
246 os.remove(spirv)
247
Robert Konradcec9c702017-01-26 09:45:17 +0100248def test_shader_hlsl(stats, shader, update, keep):
249 joined_path = os.path.join(shader[0], shader[1])
250 print('Testing HLSL shader:', joined_path)
251 spirv, msl = cross_compile_hlsl(joined_path)
252 regression_check(shader, msl, update, keep)
253 os.remove(spirv)
254
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100255def test_shaders_helper(stats, shader_dir, update, malisc, keep, backend):
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200256 for root, dirs, files in os.walk(os.path.join(shader_dir)):
257 for i in files:
258 path = os.path.join(root, i)
259 relpath = os.path.relpath(path, shader_dir)
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100260 if backend == 'metal':
261 test_shader_msl(stats, (shader_dir, relpath), update, keep)
Robert Konradcec9c702017-01-26 09:45:17 +0100262 elif backend == 'hlsl':
263 test_shader_hlsl(stats, (shader_dir, relpath), update, keep)
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100264 else:
265 test_shader(stats, (shader_dir, relpath), update, keep)
Hans-Kristian Arntzen78c9a802016-05-11 19:39:38 +0200266
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100267def test_shaders(shader_dir, update, malisc, keep, backend):
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100268 if malisc:
269 with open('stats.csv', 'w') as stats:
270 print('Shader,OrigRegs,OrigUniRegs,OrigALUShort,OrigLSShort,OrigTEXShort,OrigALULong,OrigLSLong,OrigTEXLong,CrossRegs,CrossUniRegs,CrossALUShort,CrossLSShort,CrossTEXShort,CrossALULong,CrossLSLong,CrossTEXLong', file = stats)
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100271 test_shaders_helper(stats, shader_dir, update, malisc, keep, backend)
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100272 else:
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100273 test_shaders_helper(None, shader_dir, update, malisc, keep, backend)
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100274
275def main():
276 parser = argparse.ArgumentParser(description = 'Script for regression testing.')
277 parser.add_argument('folder',
278 help = 'Folder containing shader files to test.')
279 parser.add_argument('--update',
280 action = 'store_true',
281 help = 'Updates reference files if there is a mismatch. Use when legitimate changes in output is found.')
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100282 parser.add_argument('--keep',
283 action = 'store_true',
284 help = 'Leave failed GLSL shaders on disk if they fail regression. Useful for debugging.')
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100285 parser.add_argument('--malisc',
286 action = 'store_true',
Hans-Kristian Arntzen147e53a2016-04-04 09:36:04 +0200287 help = 'Use malisc offline compiler to determine static cycle counts before and after spirv-cross.')
Hans-Kristian Arntzen1c28ec62017-01-18 19:05:57 +0100288 parser.add_argument('--metal',
289 action = 'store_true',
290 help = 'Test Metal backend.')
Robert Konradcec9c702017-01-26 09:45:17 +0100291 parser.add_argument('--hlsl',
292 action = 'store_true',
293 help = 'Test HLSL backend.')
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100294 args = parser.parse_args()
295
296 if not args.folder:
297 sys.stderr.write('Need shader folder.\n')
298 sys.exit(1)
299
Robert Konradcec9c702017-01-26 09:45:17 +0100300 test_shaders(args.folder, args.update, args.malisc, args.keep, 'metal' if args.metal else ('hlsl' if args.hlsl else 'glsl'))
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100301 if args.malisc:
302 print('Stats in stats.csv!')
Hans-Kristian Arntzene50e04c2016-03-22 14:56:50 +0100303 print('Tests completed!')
Hans-Kristian Arntzen75471fb2016-03-02 18:09:16 +0100304
305if __name__ == '__main__':
Hans-Kristian Arntzen0a5b3a62016-03-22 14:44:12 +0100306 main()