Gitiles
Code Review Sign In
gerrit.openfyde.cn / chromium.googlesource.com / external / github.com / cisco / openh264 / 7175d3c90e63d2c5adc87a92ed3625fcd5d5e1a4 / . / docs / doxygen2rst.py
blob: df91b10329c5fc4f0927a9029b9a0691fbd1f0c2 [file] [log] [blame]
huashi7175d3c2014-11-21 10:38:30 +0800[diff] [blame^]1from __future__ import print_function
2import re, sys, os, time, glob, errno, tempfile, binascii, subprocess, shutil
3from lxml import etree
4from optparse import OptionParser
5import textwrap
6import string
7
8VERSION = '0.1'
9__all__ = ['DoxyGen2RST']
10LINE_BREAKER = "\n"
11MAX_COLUMN = 80
12
13def is_valid_uuid(uuid_string):
14 uuid4hex = re.compile('[0-9a-f]{32}\Z', re.I)
15 return uuid4hex.match(uuid_string) != None
16
17def get_page(refid):
18 fields = refid.split("_")
19 if(is_valid_uuid(fields[-1][-32:])):
20 return ["_".join(fields[0:-1]), fields[-1]]
21 return [refid, None]
22
23def mkdir_p(path):
24 try:
25 os.makedirs(path)
26 except OSError as exc: # Python >2.5
27 if exc.errno == errno.EEXIST and os.path.isdir(path):
28 pass
29 else:
30 raise
31
32def _glob(path, *exts):
33 path = os.path.join(path, "*") if os.path.isdir(path) else path + "*"
34 return [f for files in [glob.glob(path + ext) for ext in exts] for f in files]
35
36class DoxyGen2RST(object):
37 """
38 Customize the Doxygen XML output into RST format, then it can
39 be translated into all formats with the unified user interface.
40 The Doxygen output itself is too verbose and not hard to be
41 organized for a good documentation.
42 """
43
44 def __init__(self,
45 src,
46 dst,
47 missing_filename = "missing.rst",
48 is_github = False,
49 enable_uml = True,
50 github_ext = ""):
51 self.doxy_output_dir = os.path.join(src, "_doxygen", "xml")
52 self.output_dir = dst
53 self.rst_dir = src
54 self.enable_uml = enable_uml
55 mkdir_p(dst)
56 self.is_github = is_github
57 if(is_github):
58 self.page_ext = github_ext
59 self.anchor_prefix = "wiki-"
60 else:
61 self.anchor_prefix = ""
62 self.page_ext = ".html"
63 self.filter = ["*.rst", "*.rest"]
64 self.re_doxy = "<doxygen2rst\s(\S*)=(\S*)>(.*?)</doxygen2rst>"
65 self.index_root = etree.parse(os.path.join(self.doxy_output_dir, "index.xml")).getroot()
66 self.references = {}
67 self.missed_types_structs = {}
68 self.name_refid_map = {}
69 self.build_references()
70 self.page_references = {}
71 self.missing_filename = missing_filename
72 self.temp_uml_path = os.path.join(tempfile.gettempdir(), "uml_" + binascii.b2a_hex(os.urandom(15)))
73 if os.path.exists(self.temp_uml_path):
74 shutil.rmtree(self.temp_uml_path)
75 os.mkdir(self.temp_uml_path)
76
77 def _find_ref_id(self, kind, name):
78 #print("_find_ref_id, %s - %s" %(kind, name))
79 if(kind == "function"):
80 for comp in self.index_root.iter("member"):
81 if(comp.attrib["kind"].lower() == kind.lower() and
82 comp.findtext("name").lower() == name.lower()):
83 return (comp.attrib["refid"])
84 pass
85 else:
86 for comp in self.index_root.iter("compound"):
87 if(comp.attrib["kind"].lower() == kind.lower() and
88 comp.findtext("name").lower() == name.lower()):
89 return comp.attrib["refid"]
90 return None
91
92 def strip_title_ref(self, text):
93 table = string.maketrans("","")
94 retstr = text.translate(table, string.punctuation)
95 words = retstr.split()
96 retstr = "-".join(words)
97 return retstr.lower()
98
99 def build_references(self):
100 for file in _glob(self.rst_dir, *self.filter):
101 filename = os.path.basename(file)
102 fin = open(file,'r')
103 content = fin.read()
104 it = re.finditer(self.re_doxy, content, re.DOTALL)
105 for m in it:
106 ref_id = self._find_ref_id(m.groups()[0], m.groups()[1])
107 if(ref_id is None):
108 #print("Reference is NOT found for: %s=%s" % (m.groups()[0], m.groups()[1]))
109 continue
110 page_name = os.path.splitext(filename)[0]
111 title_ref = self.strip_title_ref(m.groups()[2])
112 self.references[ref_id] = [m.groups()[0], m.groups()[1], page_name, filename, title_ref]
113 self.name_refid_map[m.groups()[1]] = ref_id
114 fin.close()
115 #print(self.references)
116
117 def call_plantuml(self):
118 if(not self.enable_uml):
119 return
120
121 java_bin = os.path.join(os.environ['JAVA_HOME'], "bin", "java")
122 output_path = os.path.abspath(os.path.join(self.output_dir, "images"))
123 cmds = ["\"" + java_bin + "\"", "-jar", "plantuml.jar", self.temp_uml_path + "/", "-o", output_path]
124 print(" ".join(cmds))
125 os.system(" ".join(cmds))
126 shutil.rmtree(self.temp_uml_path)
127
128 def _build_uml(self, uml_name, content):
129 uml_path = os.path.join(self.temp_uml_path, uml_name + ".txt")
130 fuml = open(uml_path, "w+")
131 fuml.write("@startuml\n")
132 fuml.write(content)
133 fuml.write("\n@enduml\n")
134 fuml.close()
135 return ".. image:: images/" + uml_name + ".png" + LINE_BREAKER
136
137 def _build(self, m):
138 retstr = ""
139 if(m.groups()[0] == "uml"):
140 retstr = self._build_uml(m.groups()[1], m.groups()[2])
141 elif(m.groups()[0] == "link"):
142 link = m.groups()[1] + self.page_ext
143 retstr = ("`%s <%s>`_" % (m.groups()[2], link))
144 else:
145 if(m.groups()[0] != "function"):
146 retstr += self._build_title(m.groups()[2])
147 retstr += self.convert_doxy(m.groups()[0], m.groups()[1])
148
149 return retstr
150
151 def generate(self):
152 for file in _glob(self.rst_dir, *self.filter):
153 filename = os.path.basename(file)
154 fin = open(file,'r')
155 input_txt = fin.read()
156 fin.close()
157
158 output_txt = re.sub(self.re_doxy, self._build, input_txt, 0, re.DOTALL)
159 output_txt += self._build_page_ref_notes()
160
161 fout = open(os.path.join(self.output_dir, filename), 'w+')
162 fout.write(output_txt)
163 fout.close()
164 #print("%s --- %s" %( file, os.path.join(self.output_dir, filename)))
165
166 self._build_missed_types_and_structs()
167 self.call_plantuml()
168
169 def make_para_title(self, title, indent = 4):
170 retstr = LINE_BREAKER
171 if(title):
172 retstr += "".ljust(indent, " ") + "| **" + title + "**" + LINE_BREAKER
173 return retstr
174
175 def _build_title(self, title, flag = '=', ref = None):
176 retstr = LINE_BREAKER
177 if(ref):
178 retstr += ".. _ref-" + ref + ":" + LINE_BREAKER + LINE_BREAKER
179 retstr += title + LINE_BREAKER
180 retstr += "".ljust(20, flag) + LINE_BREAKER
181 retstr += LINE_BREAKER
182 return retstr
183
184 def _build_ref(self, node):
185 text = node.text.strip()
186 retstr = ""
187 target = '`' + text + '`'
188 retstr += target + "_ "
189 if target in self.page_references:
190 reflink = self.page_references[target]
191 print("Link already added: %s == %s" % (reflink[0], node.attrib["refid"]))
192 assert(reflink[0] == node.attrib["refid"])
193 pass
194 else:
195 self.page_references[target] = (node.attrib["refid"], node.attrib["kindref"], text)
196
197 return retstr
198
199 def _build_code_block(self, node):
200 retstr = "::" + LINE_BREAKER + LINE_BREAKER
201 for codeline in node.iter("codeline"):
202 retstr += " "
203 for phrases in codeline.iter("highlight"):
204 if(phrases.text):
205 retstr += phrases.text.strip()
206 for child in phrases:
207 if(child.text):
208 retstr += child.text.strip()
209 if(child.tag == "sp"):
210 retstr += " "
211 if(child.tag == "ref" and child.text):
212 #escape the reference in the code block
213 retstr += "" # self._build_ref(child)
214 if(child.tail):
215 retstr += child.tail.strip()
216 retstr += LINE_BREAKER
217 return retstr
218
219 def _build_itemlist(self, node):
220 retstr = ""
221 for para in node:
222 if(para.tag != "para"):
223 continue
224 if(para.text):
225 retstr += para.text.strip()
226 for child in para:
227 if(child.tag == "ref" and child.text):
228 retstr += self._build_ref(child)
229 if(child.tail):
230 retstr += child.tail.strip()
231
232 return retstr
233
234 def _build_itemizedlist(self, node):
235 retstr = LINE_BREAKER
236 if(node == None):
237 return ""
238 for item in node:
239 if(item.tag != "listitem"):
240 continue
241 retstr += " - " + self._build_itemlist(item)
242 retstr += LINE_BREAKER
243 return retstr
244
245 def _build_verbatim(self, node):
246 retstr = LINE_BREAKER
247 if(node.text):
248 lines = node.text.splitlines()
249 print(lines[0])
250 m = re.search("{plantuml}\s(\S*)", lines[0])
251 if(m):
252 uml_name = "uml_" + m.groups()[0]
253 retstr += self._build_uml(uml_name, "\n".join(lines[1:]))
254 else:
255 retstr += "::" + LINE_BREAKER + LINE_BREAKER
256 retstr += node.text
257
258 return retstr
259
260 def _build_para(self, para):
261 retstr = ""
262 no_new_line = False
263 if(para.text):
264 retstr += textwrap.fill(para.text.strip(), MAX_COLUMN) + LINE_BREAKER + LINE_BREAKER
265 for child in para:
266 no_new_line = False
267 if(child.tag == "simplesect"):
268 for child_para in child:
269 if(child.attrib["kind"] == "return"):
270 return_str = self._build_para(child_para)
271 retstr += "".ljust(4, " ") + "| Return:" + LINE_BREAKER
272 for line in return_str.splitlines():
273 retstr += "".ljust(4, " ") + "| " + line + LINE_BREAKER
274 elif(child_para.tag == "title" and child_para.text):
275 lf.make_para_title(child_para.text.strip(), 4)
276 elif(child_para.tag == "para"): #for @see
277 retstr += self._build_para(child_para)
278 elif(child_para.text):
279 retstr += "".ljust(4, " ") + "| " + child_para.text.strip() + LINE_BREAKER
280 if(child.tag == "preformatted"):
281 retstr += "::" + LINE_BREAKER + LINE_BREAKER
282 if(child.text):
283 for line in child.text.splitlines():
284 retstr += " " + line + LINE_BREAKER
285 if(child.tag == "ref" and child.text):
286 retstr = retstr.rstrip('\n')
287 retstr += " " + self._build_ref(child)
288 no_new_line = True
289 if(child.tag == "programlisting"):
290 retstr += self._build_code_block(child)
291 if(child.tag == "itemizedlist"):
292 retstr += self._build_itemizedlist(child)
293 if(child.tag == "verbatim"):
294 retstr += self._build_verbatim(child)
295 if(not no_new_line):
296 retstr += LINE_BREAKER
297 if(child.tail):
298 retstr += textwrap.fill(child.tail.strip(), MAX_COLUMN) + LINE_BREAKER + LINE_BREAKER
299 return retstr
300
301 def get_text(self, node):
302 retstr = ""
303 if(node == None):
304 return ""
305 for para in node:
306 if(para.tag != "para"):
307 continue
308 retstr += self._build_para(para)
309
310 return retstr
311
312 def _find_text_ref(self, node):
313 retstr = ""
314 if(node.text):
315 retstr += node.text.strip()
316 for child in node:
317 if(child.tag == "ref"):
318 retstr += " " + self._build_ref(child) + " "
319 if(child.tail):
320 retstr += child.tail.strip()
321 return retstr
322
323 def _build_row_breaker(self, columns):
324 retstr = "+"
325 for column in columns:
326 retstr += "".ljust(column, "-") + "+"
327 return retstr + LINE_BREAKER
328
329 def _wrap_cell(self, text, length = 30):
330 newlines = []
331 for line in text.splitlines():
332 newlines.extend(textwrap.wrap(line, length))
333 return newlines
334
335 def _build_row(self, row, columns):
336 retstr = ""
337 row_lines = []
338 max_line = 0
339 for i in range(3):
340 row_lines.append(row[i].splitlines())
341 if(max_line < len(row_lines[i])):
342 max_line = len(row_lines[i])
343
344 for i in range(max_line):
345 for j in range(3):
346 retstr += "|"
347 if(len(row_lines[j]) > i):
348 retstr += row_lines[j][i]
349 retstr += "".ljust(columns[j] - len(row_lines[j][i]), " ")
350 else:
351 retstr += "".ljust(columns[j], " ")
352 retstr += "|" + LINE_BREAKER
353 return retstr
354
355 def _build_table(self, rows):
356 retstr = ""
357 columns = [0, 0, 0]
358 for row in rows:
359 for i in range(3):
360 for rowline in row[i].splitlines():
361 if(columns[i] < len(rowline) + 2):
362 columns[i] = len(rowline) + 2
363
364 #columns[0] = 40 if(columns[0] > 40) else columns[0]
365 #columns[1] = 40 if(columns[1] > 40) else columns[1]
366 #columns[2] = MAX_COLUMN - columns[0] - columns[1]
367
368 retstr += self._build_row_breaker(columns)
369 for row in rows:
370 retstr += self._build_row(row, columns)
371 retstr += self._build_row_breaker(columns)
372 return retstr;
373
374 def build_param_list(self, params, paramdescs):
375 retstr = ""
376 param_descriptions = []
377 for desc in paramdescs:
378 param_descriptions.append(desc)
379
380 rows = []
381 rows.append(("Name", "Type", "Descritpion"))
382 for param in params:
383 declname = param.findtext("declname")
384 paramdesc = None
385 for desc in param_descriptions:
386 paramname = desc.findtext("parameternamelist/parametername")
387 if(paramname.lower() == declname.lower()):
388 paramdesc = desc.find("parameterdescription")
389 break
390 decltype = self._find_text_ref(param.find("type"))
391 rows.append((declname, decltype, self.get_text(paramdesc)))
392
393 if(len(rows) > 1):
394 retstr += self._build_table(rows)
395 return retstr
396
397 def _build_enum(self, member):
398 enum_id = member.attrib["id"]
399 file, tag = get_page(enum_id)
400 retstr = self._build_title(member.findtext("name"), ref = tag)
401 detail_node = self.get_desc_node(member)
402 if(detail_node is not None):
403 retstr += LINE_BREAKER
404 retstr += self.get_text(detail_node)
405
406 rows = []
407 rows.append(("Name", "Initializer", "Descritpion"))
408 for enumvalue in member.iter("enumvalue"):
409 name = enumvalue.findtext("name")
410 initializer = enumvalue.findtext("initializer")
411 if(not initializer):
412 initializer = ""
413 desc = self.get_text(enumvalue.find("briefdescription"))
414 desc += self.get_text(enumvalue.find("detaileddescription"))
415 if(not desc):
416 desc = ""
417 rows.append((name, initializer, desc))
418
419 if(len(rows) > 1):
420 retstr += self._build_table(rows)
421 return retstr
422
423
424 def _build_struct(self, node):
425 retstr = ""
426 detail_node = self.get_desc_node(node)
427 if(detail_node is not None):
428 retstr += self.get_text(detail_node) + LINE_BREAKER
429 rows = []
430 rows.append(("Name", "Type", "Descritpion"))
431 for member in node.iter("memberdef"):
432 if(member.attrib["kind"] == "variable"):
433 name = member.findtext("name")
434 type = self._find_text_ref(member.find("type"))
435 desc = self.get_text(member.find("briefdescription"))
436 desc += self.get_text(member.find("detaileddescription"))
437 desc += self.get_text(member.find("inbodydescription"))
438 if(not desc):
439 desc = ""
440 rows.append((name, type, desc))
441
442 if(len(rows) > 1):
443 retstr += self._build_table(rows)
444 return retstr
445
446 def _build_class(self, node):
447 retstr = ""
448
449 for member in node.iter("memberdef"):
450 if(member.attrib["kind"] == "function"):
451 retstr += self.build_function(member)
452 return retstr
453
454 def get_desc_node(self, member):
455 detail_node = member.find("detaileddescription")
456 brief_node = member.find("briefdescription")
457 detail_txt = ""
458 if(detail_node == None and brief_node == None):
459 return None
460
461 if(detail_node is not None):
462 detail_txt = detail_node.findtext("para")
463
464 if(not detail_txt and brief_node != None):
465 detail_txt = brief_node.findtext("para")
466 detail_node = brief_node
467
468 return detail_node
469
470 def build_function(self, member):
471 retstr = ""
472
473 desc_node = self.get_desc_node(member)
474 if(desc_node is None):
475 return ""
476 detail_txt = desc_node.findtext("para")
477 if(not detail_txt or detail_txt.strip() == "{ignore}"):
478 return ""
479
480 func_id = member.attrib["id"]
481 page_id, ref_id = get_page(func_id)
482 retstr += self._build_title(member.findtext("name"), '-', ref = ref_id)
483 retstr += self.get_text(desc_node)
484 retstr += LINE_BREAKER
485 detail_node = member.find("detaileddescription")
486 if(desc_node != detail_node):
487 retstr += self.get_text(detail_node)
488 retstr += self.build_param_list(member.iter("param"), detail_node.iter("parameteritem"))
489 return retstr
490
491 def _build_missed_types_and_structs(self):
492 fout = open(os.path.join(self.output_dir, self.missing_filename), 'w+')
493 fout.write(".. contents:: " + LINE_BREAKER)
494 fout.write(" :local:" + LINE_BREAKER)
495 fout.write(" :depth: 2" + LINE_BREAKER + LINE_BREAKER)
496
497 footnote = ""
498 while (len(self.missed_types_structs) > 0):
499 for key, value in self.missed_types_structs.iteritems():
500 fout.write(self.covert_item(value[0], key, value[1]))
501 #print(value)
502 self.missed_types_structs = {}
503 footnote += self._build_page_ref_notes()
504
505 fout.write(footnote)
506
507 fout.close()
508
509 def _build_page_ref_notes(self):
510 retstr = LINE_BREAKER
511 #TODO
512 for key, value in self.page_references.iteritems():
513 page, tag = get_page(value[0])
514 m = re.search("_8h_", page)
515 if(m):
516 continue;
517
518 rstname = None
519 anchor = value[2].lower()
520 if not page in self.references:
521 self.missed_types_structs[value[0]] = (page, tag)
522 rstname = os.path.splitext(self.missing_filename)[0]
523 else:
524 rstname = self.references[page][2]
525 anchor = self.references[page][4]
526 #if(tag and not self.is_github):
527 # anchor = self.anchor_prefix + "ref-" + tag
528 retstr += ".. _" + key + ": " + rstname + self.page_ext + "#" + anchor
529 retstr += LINE_BREAKER + LINE_BREAKER
530 self.page_references = {}
531 return retstr
532
533 def _build_item_by_id(self, node, id):
534 retstr = ""
535 for member in node.iter("memberdef"):
536 if(member.attrib["id"] != id):
537 continue
538 if(member.attrib["kind"] == "enum"):
539 retstr += self._build_enum(member)
540 return retstr
541
542 def covert_item(self, compound, id, tag):
543 xml_path = os.path.join(self.doxy_output_dir, "%s.xml" % compound)
544 print("covert_item: id=%s, name=%s" % (id, xml_path))
545 obj_root = etree.parse(xml_path).getroot()
546 retstr = ""
547 compound = obj_root.find("compounddef")
548 compound_kind = compound.attrib["kind"]
549 if(not tag):
550 retstr += self._build_title(compound.findtext("compoundname"))
551 if(compound_kind == "class"):
552 retstr += self._build_class(compound)
553 elif(compound_kind == "struct"):
554 retstr += self._build_struct(compound)
555 else:
556 retstr += self._build_item_by_id(compound, id)
557
558 return retstr
559
560 def _build_page(self, compound):
561 retstr = ""
562 retstr += self.get_text(compound.find("detaileddescription"))
563 return retstr
564
565 def _build_file(self, compound, type, ref_id, name):
566 retstr = ""
567 for member in compound.iter("memberdef"):
568 if(member.attrib["kind"] == "function" and member.attrib["id"] == ref_id):
569 retstr += self.build_function(member)
570 return retstr
571
572 def convert_doxy(self, type, name):
573 #print(name)
574 file = ref_id = self.name_refid_map[name]
575 dst_kind = type
576 if(type == "function"):
577 file, tag = get_page(ref_id)
578 dst_kind = "file"
579 xml_path = os.path.join(self.doxy_output_dir, "%s.xml" % file)
580 print("convert_doxy: type=%s, name=%s" % (type, xml_path))
581 obj_root = etree.parse(xml_path).getroot()
582 compound = obj_root.find("compounddef")
583 compound_kind = compound.attrib["kind"]
584 assert(dst_kind == compound_kind)
585 retstr = ""
586 if(compound_kind == "class"):
587 retstr += self._build_class(compound)
588 elif(compound_kind == "struct"):
589 retstr += self._build_struct(compound)
590 elif(compound_kind == "page"):
591 retstr += self._build_page(compound)
592 elif(compound_kind == "group"):
593 retstr += self._build_page(compound)
594 elif(compound_kind == "file"):
595 retstr += self._build_file(compound, type, ref_id, name)
596 return retstr
597
598
599if __name__ == '__main__':
600 import argparse
601 parser = argparse.ArgumentParser()
602 parser.add_argument("-g", "--github", action="store_true", help="Render the link in format of github wiki.")
603 parser.add_argument("-e", "--ext", default="", help="extension for github wiki")
604 parser.add_argument("-i", "--input", default="doxygen", help="Input file path of doxygen output and source rst file.")
605 parser.add_argument("-o", "--output", default="wikipage", help="Output converted restructured text files to path.")
606 parser.add_argument("-s", "--struct", default="TypesAndStructures.rest", help="Output of auto generated enum and structures.")
607 parser.add_argument("-u", "--uml", action="store_true", help="Enable UML, you need to download plantuml.jar from Plantuml and put it to here. http://plantuml.sourceforge.net/")
608
609 args = parser.parse_args()
610 ext = ""
611 if(len(args.ext) > 0):
612 ext = ("." + args.ext)
613 agent = DoxyGen2RST(args.input,
614 args.output,
615 args.struct,
616 is_github = True,
617 enable_uml = args.uml,
618 github_ext = ext)
619 agent.generate()