blob: 3500d9ce5e0c19a2b4a8a7a7290f52c10253378b [file] [log] [blame]
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +02001#!/usr/bin/env python3
2# SPDX-License-Identifier: LGPL-2.1+
3
Zbigniew Jędrzejewski-Szmek0f5cea02020-08-27 19:27:18 +02004import argparse
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +02005import collections
6import sys
Zbigniew Jędrzejewski-Szmekc351d562020-04-24 12:09:07 +02007import os
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +02008import shlex
9import subprocess
10import io
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +020011from lxml import etree
12
13PARSER = etree.XMLParser(no_network=True,
14 remove_comments=False,
15 strip_cdata=False,
16 resolve_entities=False)
17
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +020018class NoCommand(Exception):
19 pass
20
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +020021BORING_INTERFACES = [
22 'org.freedesktop.DBus.Peer',
23 'org.freedesktop.DBus.Introspectable',
24 'org.freedesktop.DBus.Properties',
25]
26
27def print_method(declarations, elem, *, prefix, file, is_signal=False):
28 name = elem.get('name')
29 klass = 'signal' if is_signal else 'method'
30 declarations[klass].append(name)
31
32 print(f'''{prefix}{name}(''', file=file, end='')
33 lead = ',\n' + prefix + ' ' * len(name) + ' '
34
35 for num, arg in enumerate(elem.findall('./arg')):
36 argname = arg.get('name')
37
38 if argname is None:
Zbigniew Jędrzejewski-Szmek04aa6fa2020-08-27 20:15:30 +020039 if opts.print_errors:
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +020040 print(f'method {name}: argument {num+1} has no name', file=sys.stderr)
41 argname = 'UNNAMED'
42
43 type = arg.get('type')
44 if not is_signal:
45 direction = arg.get('direction')
46 print(f'''{lead if num > 0 else ''}{direction:3} {type} {argname}''', file=file, end='')
47 else:
48 print(f'''{lead if num > 0 else ''}{type} {argname}''', file=file, end='')
49
50 print(f');', file=file)
51
52ACCESS_MAP = {
53 'read' : 'readonly',
54 'write' : 'readwrite',
55}
56
57def value_ellipsis(type):
58 if type == 's':
59 return "'...'";
60 if type[0] == 'a':
61 inner = value_ellipsis(type[1:])
62 return f"[{inner}{', ...' if inner != '...' else ''}]";
63 return '...'
64
65def print_property(declarations, elem, *, prefix, file):
66 name = elem.get('name')
67 type = elem.get('type')
68 access = elem.get('access')
69
70 declarations['property'].append(name)
71
72 # @org.freedesktop.DBus.Property.EmitsChangedSignal("false")
73 # @org.freedesktop.systemd1.Privileged("true")
74 # readwrite b EnableWallMessages = false;
75
76 for anno in elem.findall('./annotation'):
77 anno_name = anno.get('name')
78 anno_value = anno.get('value')
79 print(f'''{prefix}@{anno_name}("{anno_value}")''', file=file)
80
81 access = ACCESS_MAP.get(access, access)
82 print(f'''{prefix}{access} {type} {name} = {value_ellipsis(type)};''', file=file)
83
Zbigniew Jędrzejewski-Szmek08fe1b62020-04-10 14:46:44 +020084def print_interface(iface, *, prefix, file, print_boring, only_interface, declarations):
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +020085 name = iface.get('name')
86
Zbigniew Jędrzejewski-Szmek08fe1b62020-04-10 14:46:44 +020087 is_boring = (name in BORING_INTERFACES or
88 only_interface is not None and name != only_interface)
89
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +020090 if is_boring and print_boring:
91 print(f'''{prefix}interface {name} {{ ... }};''', file=file)
Zbigniew Jędrzejewski-Szmek08fe1b62020-04-10 14:46:44 +020092
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +020093 elif not is_boring and not print_boring:
94 print(f'''{prefix}interface {name} {{''', file=file)
95 prefix2 = prefix + ' '
96
97 for num, elem in enumerate(iface.findall('./method')):
98 if num == 0:
99 print(f'''{prefix2}methods:''', file=file)
100 print_method(declarations, elem, prefix=prefix2 + ' ', file=file)
101
102 for num, elem in enumerate(iface.findall('./signal')):
103 if num == 0:
104 print(f'''{prefix2}signals:''', file=file)
105 print_method(declarations, elem, prefix=prefix2 + ' ', file=file, is_signal=True)
106
107 for num, elem in enumerate(iface.findall('./property')):
108 if num == 0:
109 print(f'''{prefix2}properties:''', file=file)
110 print_property(declarations, elem, prefix=prefix2 + ' ', file=file)
111
112 print(f'''{prefix}}};''', file=file)
113
114def document_has_elem_with_text(document, elem, item_repr):
115 predicate = f".//{elem}" # [text() = 'foo'] doesn't seem supported :(
116 for loc in document.findall(predicate):
117 if loc.text == item_repr:
118 return True
119 else:
120 return False
121
Zbigniew Jędrzejewski-Szmekaf4c7dc2020-08-27 19:21:21 +0200122def check_documented(document, declarations, stats):
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200123 missing = []
124 for klass, items in declarations.items():
Zbigniew Jędrzejewski-Szmekaf4c7dc2020-08-27 19:21:21 +0200125 stats['total'] += len(items)
126
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200127 for item in items:
128 if klass == 'method':
129 elem = 'function'
130 item_repr = f'{item}()'
131 elif klass == 'signal':
132 elem = 'function'
133 item_repr = item
134 elif klass == 'property':
135 elem = 'varname'
136 item_repr = item
137 else:
138 assert False, (klass, item)
139
140 if not document_has_elem_with_text(document, elem, item_repr):
Zbigniew Jędrzejewski-Szmek04aa6fa2020-08-27 20:15:30 +0200141 if opts.print_errors:
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200142 print(f'{klass} {item} is not documented :(')
143 missing.append((klass, item))
144
Zbigniew Jędrzejewski-Szmekaf4c7dc2020-08-27 19:21:21 +0200145 stats['missing'] += len(missing)
146
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200147 return missing
148
Zbigniew Jędrzejewski-Szmek08fe1b62020-04-10 14:46:44 +0200149def xml_to_text(destination, xml, *, only_interface=None):
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200150 file = io.StringIO()
151
152 declarations = collections.defaultdict(list)
Jérémy Rosenf92c8d12020-04-18 20:19:50 +0200153 interfaces = []
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200154
155 print(f'''node {destination} {{''', file=file)
156
157 for print_boring in [False, True]:
158 for iface in xml.findall('./interface'):
159 print_interface(iface, prefix=' ', file=file,
160 print_boring=print_boring,
Zbigniew Jędrzejewski-Szmek08fe1b62020-04-10 14:46:44 +0200161 only_interface=only_interface,
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200162 declarations=declarations)
Jérémy Rosenf92c8d12020-04-18 20:19:50 +0200163 name = iface.get('name')
164 if not name in BORING_INTERFACES:
165 interfaces.append(name)
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200166
167 print(f'''}};''', file=file)
168
Jérémy Rosenf92c8d12020-04-18 20:19:50 +0200169 return file.getvalue(), declarations, interfaces
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200170
Zbigniew Jędrzejewski-Szmekaf4c7dc2020-08-27 19:21:21 +0200171def subst_output(document, programlisting, stats):
Zbigniew Jędrzejewski-Szmekc351d562020-04-24 12:09:07 +0200172 executable = programlisting.get('executable', None)
173 if executable is None:
174 # Not our thing
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200175 return
Zbigniew Jędrzejewski-Szmekc351d562020-04-24 12:09:07 +0200176 executable = programlisting.get('executable')
177 node = programlisting.get('node')
178 interface = programlisting.get('interface')
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200179
Zbigniew Jędrzejewski-Szmek0f5cea02020-08-27 19:27:18 +0200180 argv = [f'{opts.build_dir}/{executable}', f'--bus-introspect={interface}']
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200181 print(f'COMMAND: {shlex.join(argv)}')
182
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200183 try:
184 out = subprocess.check_output(argv, text=True)
Zbigniew Jędrzejewski-Szmekc351d562020-04-24 12:09:07 +0200185 except FileNotFoundError:
186 print(f'{executable} not found, ignoring', file=sys.stderr)
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200187 return
188
189 xml = etree.fromstring(out, parser=PARSER)
190
Zbigniew Jędrzejewski-Szmekc351d562020-04-24 12:09:07 +0200191 new_text, declarations, interfaces = xml_to_text(node, xml, only_interface=interface)
192 programlisting.text = '\n' + new_text + ' '
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200193
194 if declarations:
Zbigniew Jędrzejewski-Szmekaf4c7dc2020-08-27 19:21:21 +0200195 missing = check_documented(document, declarations, stats)
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200196 parent = programlisting.getparent()
197
198 # delete old comments
199 for child in parent:
200 if (child.tag == etree.Comment
Jérémy Rosenf92c8d12020-04-18 20:19:50 +0200201 and 'Autogenerated' in child.text):
202 parent.remove(child)
203 if (child.tag == etree.Comment
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200204 and 'not documented' in child.text):
205 parent.remove(child)
Jérémy Rosenf92c8d12020-04-18 20:19:50 +0200206 if (child.tag == "variablelist"
207 and child.attrib.get("generated",False) == "True"):
208 parent.remove(child)
209
210 # insert pointer for systemd-directives generation
211 the_tail = programlisting.tail #tail is erased by addnext, so save it here.
212 prev_element = etree.Comment("Autogenerated cross-references for systemd.directives, do not edit")
213 programlisting.addnext(prev_element)
214 programlisting.tail = the_tail
215
216 for interface in interfaces:
217 variablelist = etree.Element("variablelist")
218 variablelist.attrib['class'] = 'dbus-interface'
219 variablelist.attrib['generated'] = 'True'
220 variablelist.attrib['extra-ref'] = interface
221
222 prev_element.addnext(variablelist)
223 prev_element.tail = the_tail
224 prev_element = variablelist
225
226 for decl_type,decl_list in declarations.items():
227 for declaration in decl_list:
228 variablelist = etree.Element("variablelist")
229 variablelist.attrib['class'] = 'dbus-'+decl_type
230 variablelist.attrib['generated'] = 'True'
231 if decl_type == 'method' :
232 variablelist.attrib['extra-ref'] = declaration + '()'
233 else:
234 variablelist.attrib['extra-ref'] = declaration
235
236 prev_element.addnext(variablelist)
237 prev_element.tail = the_tail
238 prev_element = variablelist
239
240 last_element = etree.Comment("End of Autogenerated section")
241 prev_element.addnext(last_element)
242 prev_element.tail = the_tail
243 last_element.tail = the_tail
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200244
245 # insert comments for undocumented items
246 for item in reversed(missing):
247 comment = etree.Comment(f'{item[0]} {item[1]} is not documented!')
248 comment.tail = programlisting.tail
249 parent.insert(parent.index(programlisting) + 1, comment)
250
251def process(page):
252 src = open(page).read()
253 xml = etree.fromstring(src, parser=PARSER)
254
255 # print('parsing {}'.format(name), file=sys.stderr)
256 if xml.tag != 'refentry':
257 return
258
Zbigniew Jędrzejewski-Szmekaf4c7dc2020-08-27 19:21:21 +0200259 stats = collections.Counter()
260
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200261 pls = xml.findall('.//programlisting')
262 for pl in pls:
Zbigniew Jędrzejewski-Szmekaf4c7dc2020-08-27 19:21:21 +0200263 subst_output(xml, pl, stats)
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200264
265 out_text = etree.tostring(xml, encoding='unicode')
Frantisek Sumsal86b52a32020-04-21 20:46:53 +0200266 # massage format to avoid some lxml whitespace handling idiosyncrasies
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200267 # https://bugs.launchpad.net/lxml/+bug/526799
268 out_text = (src[:src.find('<refentryinfo')] +
269 out_text[out_text.find('<refentryinfo'):] +
270 '\n')
271
Zbigniew Jędrzejewski-Szmek1b584f32020-08-27 19:55:55 +0200272 if not opts.test:
273 with open(page, 'w') as out:
274 out.write(out_text)
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200275
Zbigniew Jędrzejewski-Szmek1b584f32020-08-27 19:55:55 +0200276 return dict(stats=stats, outdated=(out_text != src))
Zbigniew Jędrzejewski-Szmekaf4c7dc2020-08-27 19:21:21 +0200277
Zbigniew Jędrzejewski-Szmek0f5cea02020-08-27 19:27:18 +0200278def parse_args():
279 p = argparse.ArgumentParser()
Zbigniew Jędrzejewski-Szmek1b584f32020-08-27 19:55:55 +0200280 p.add_argument('--test', action='store_true',
281 help='only verify that everything is up2date')
Zbigniew Jędrzejewski-Szmek0f5cea02020-08-27 19:27:18 +0200282 p.add_argument('--build-dir', default='build')
283 p.add_argument('pages', nargs='+')
Zbigniew Jędrzejewski-Szmek04aa6fa2020-08-27 20:15:30 +0200284 opts = p.parse_args()
285 opts.print_errors = not opts.test
286 return opts
Zbigniew Jędrzejewski-Szmek0f5cea02020-08-27 19:27:18 +0200287
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200288if __name__ == '__main__':
Zbigniew Jędrzejewski-Szmek0f5cea02020-08-27 19:27:18 +0200289 opts = parse_args()
Zbigniew Jędrzejewski-Szmeke5dd26c2020-04-07 16:58:58 +0200290
Zbigniew Jędrzejewski-Szmek0f5cea02020-08-27 19:27:18 +0200291 if not os.path.exists(f'{opts.build_dir}/systemd'):
292 exit(f"{opts.build_dir}/systemd doesn't exist. Use --build-dir=.")
Zbigniew Jędrzejewski-Szmekc351d562020-04-24 12:09:07 +0200293
Zbigniew Jędrzejewski-Szmek0f5cea02020-08-27 19:27:18 +0200294 stats = {page.split('/')[-1] : process(page) for page in opts.pages}
Zbigniew Jędrzejewski-Szmekaf4c7dc2020-08-27 19:21:21 +0200295
296 # Let's print all statistics at the end
297 mlen = max(len(page) for page in stats)
Zbigniew Jędrzejewski-Szmek1b584f32020-08-27 19:55:55 +0200298 total = sum((item['stats'] for item in stats.values()), start=collections.Counter())
299 total = 'total', dict(stats=total, outdated=False)
300 outdated = []
301 for page, info in sorted(stats.items()) + [total]:
302 m = info['stats']['missing']
303 t = info['stats']['total']
Zbigniew Jędrzejewski-Szmekaf4c7dc2020-08-27 19:21:21 +0200304 p = page + ':'
Zbigniew Jędrzejewski-Szmek1b584f32020-08-27 19:55:55 +0200305 c = 'OUTDATED' if info['outdated'] else ''
306 if c:
307 outdated.append(page)
308 print(f'{p:{mlen + 1}} {t - m}/{t} {c}')
309
310 if opts.test and outdated:
Zbigniew Jędrzejewski-Szmekc91e3112020-08-27 20:18:05 +0200311 exit(f'Outdated pages: {", ".join(outdated)}\n'
312 f'Hint: ninja -C {opts.build_dir} man/update-dbus-docs')