2 from latexparser import latexparser, TexCmd
3 import distutils.dep_util
5 import cPickle as pickle
8 from qfile import QOpen
9 from string import Template
11 # useful things for pyparsing
13 def listify(s, loc, toks):
15 x.setParseAction(listify)
18 def listify(s, loc, toks):
20 x.setParseAction(listify)
23 return returnList(pp.Optional(word + pp.ZeroOrMore(pp.Suppress(',') + word)))
25 return pp.Suppress(pp.Literal(s))
29 python_api = pythonapi.reader("../../interfaces/python/api")
33 def __init__(self, filename, language, abspath):
34 assert language in ['py', 'c', 'cpp']
35 self.language = language
37 self.abspath = abspath
38 os.path.abspath(os.path.dirname(filename))
40 self.f_index = QOpen(os.path.join(self.language, filename), 'wt')
48 self.errors = open('errors.%s' % language, 'wt')
49 self.unhandled_commands = set()
51 self.function_props = {}
52 self.covered = set() # covered functions, used for error report
56 self.freshline = len(s) > 0 and (s[-1] == '\n')
57 self.f.write(s.replace('\n', '\n' + self.indent * " "))
59 def appendspace(self):
60 """ append a space to the output - if we're not at the start of a line """
61 if not self.freshline:
65 if (len(s) > 1) and (s[0] == '$' and s[-1] == '$') and self.state != 'math':
66 s = ":math:`%s`" % s[1:-1].strip()
67 elif self.state != 'math':
69 if len(s) > 0 and s[-1] == '\n':
71 if self.state == 'fpreamble':
77 if self.state == 'math':
89 meth = self.cmd_gomath
91 cname = "cmd_" + c.cmd
92 meth = getattr(self, cname, self.unrecognized_cmd)
95 def cmd_gomath(self, c):
97 print >>self, "\n\n.. math::"
101 def cmd_chapter(self, c):
102 filename = str(c.params[0]).lower().replace(' ', '_').replace('/','_')
103 self.f_index.write(" %s\n" % filename)
104 self.f_chapter = QOpen(os.path.join(self.language, filename + '.rst'), 'wt')
105 self.f_section = None
106 self.f = self.f_chapter
108 title = str(c.params[0])
109 print >>self, '*' * len(title)
111 print >>self, '*' * len(title)
113 self.chapter_intoc = False
115 def cmd_section(self, c):
116 filename = str(c.params[0]).lower().replace(' ', '_').replace('/','_')
117 if not self.chapter_intoc:
118 self.chapter_intoc = True
119 print >>self.f_chapter
120 print >>self.f_chapter, '.. toctree::'
121 print >>self.f_chapter, ' :maxdepth: 2'
122 print >>self.f_chapter
123 self.f_chapter.write(" %s\n" % filename)
124 self.f_section = QOpen(os.path.join(self.language, filename + '.rst'), 'wt')
125 self.f = self.f_section
127 title = self.render(c.params[0].str)
129 print >>self, '=' * len(title)
131 print >>self, '.. highlight:: %s' % {'c': 'c', 'cpp': 'cpp', 'py': 'python'}[self.language]
134 def cmd_subsection(self, c):
136 nm = str(c.params[0])
138 print >>self, '-' * len(nm)
140 self.function_props = {}
143 def cmd_includegraphics(self, c):
144 filename = os.path.join('..', '..', str(c.params[0]))
145 print >>self, "\n\n.. image:: %s\n\n" % filename
147 def cmd_cvCppCross(self, c):
148 self.write(":func:`%s`" % str(c.params[0]))
150 def cmd_cvCPyCross(self, c):
151 self.write(":ref:`%s`" % str(c.params[0]))
153 def cmd_cross(self, c):
154 self.write(":ref:`%s`" % str(c.params[0]))
156 def cmd_cvCross(self, c):
157 self.write(":ref:`%s`" % str(c.params[0]))
159 def cmd_cvclass(self, c):
162 nm = self.render(list(c.params[0].str))
163 print >>self, "\n.. index:: %s\n" % nm
164 print >>self, ".. _%s:\n" % nm
166 print >>self, '-' * len(nm)
168 if self.language == 'py':
169 print >>self, ".. class:: " + nm + "\n"
171 print >>self, ".. ctype:: " + nm + "\n"
176 def addtag(self, nm, c):
178 self.report_error(c, "empty name")
179 self.tags[nm] = "%s\t%s\t%d" % (nm, os.path.join(os.getcwd(), c.filename), c.lineno)
181 def cmd_cvfunc(self, c):
182 self.cmd_cvCPyFunc(c)
184 def cmd_cvCPyFunc(self, c):
186 nm = self.render(c.params[0].str)
187 print >>self, "\n.. index:: %s\n" % nm
188 print >>self, ".. _%s:\n" % nm
190 print >>self, '-' * len(nm)
192 self.state = 'fpreamble'
193 if self.description != "":
194 self.report_error(c, "overflow - preceding cvfunc (starting %s) not terminated?" % repr(self.description[:30]))
195 self.description = ""
198 self.function_props = {'name' : nm}
201 def cmd_cvCppFunc(self, c):
203 nm = self.render(c.params[0].str)
204 print >>self, "\n.. index:: %s\n" % nm
206 print >>self, "\n.. _%s:\n" % nm
208 print >>self, 'cv::%s' % nm
209 print >>self, '-' * (4+len(nm))
211 self.state = 'fpreamble'
212 if self.description != "":
213 self.report_error(c, "overflow - preceding cvfunc (starting %s) not terminated?" % repr(self.description[:30]))
214 self.description = ""
217 self.function_props = {'name' : nm}
220 def cmd_cvdefC(self, c):
221 if self.language != 'c':
223 s = str(c.params[0]).replace('\\_', '_')
224 s = s.replace('\\par', '')
225 s = s.replace('\n', ' ')
226 s = s.replace(';', '')
228 for proto in s.split('\\newline'):
229 if proto.strip() != "":
230 print >>self, "\n\n.. cfunction:: " + proto.strip() + "\n"
231 # print >>self, "=", repr(c.params[0].str)
232 print >>self, ' ' + self.description
233 self.description = ""
236 self.function_props['defpy'] = s
238 def cmd_cvdefCpp(self, c):
239 if self.language != 'cpp':
241 s = str(c.params[0]).replace('\\_', '_')
242 s = s.replace('\\par', '')
243 s = s.replace('\n', ' ')
244 s = s.replace(';', '')
246 for proto in s.split('\\newline'):
247 if proto.strip() != "":
248 print >>self, "\n\n.. cfunction:: " + proto.strip() + "\n"
249 # print >>self, "=", repr(c.params[0].str)
250 if self.description != "":
251 print >>self, ' ' + self.description
253 self.report_error(c, 'empty description')
254 self.description = ""
257 self.function_props['defpy'] = s
259 def cmd_cvdefPy(self, c):
260 if self.language != 'py':
262 s = str(c.params[0]).replace('\\_', '_')
264 print >>self, ".. function:: " + s + "\n"
265 # print >>self, "=", repr(c.params[0].str)
266 print >>self, ' ' + self.description
268 self.description = ""
270 self.function_props['defpy'] = s
272 pp.ParserElement.setDefaultWhitespaceChars(" \n\t")
274 ident = pp.Word(pp.alphanums + "_.+-")
275 ident_or_tuple = ident | (sl('(') + CommaList(ident) + sl(')'))
276 initializer = ident_or_tuple
277 arg = returnList(ident + pp.Optional(sl('=') + initializer))
279 decl = ident + sl('(') + CommaList(arg) + sl(')') + sl("->") + ident_or_tuple + pp.StringEnd()
282 l = decl.parseString(s)
283 if str(l[0]) != self.function_props['name']:
284 self.report_error(c, 'Decl "%s" does not match function name "%s"' % (str(l[0]), self.function_props['name']))
285 self.function_props['signature'] = l
286 if l[0] in python_api:
287 (ins, outs) = python_api[l[0]]
288 ins = [a for a in ins if not 'O' in a.flags]
290 outs = outs.split(',')
291 if len(ins) != len(l[1]):
292 self.report_error(c, "function %s documented arity %d, code arity %d" % (l[0], len(l[1]), len(ins)))
295 self.report_error(c, "function %s documented None, but code has %s" % (l[0], l[2]))
297 if isinstance(l[2], str):
301 if len(outs) != len(doc_outs):
302 self.report_error(c, "function %s output code tuple %d, documented %d" % (l[0], len(outs), len(doc_outs)))
304 # self.report_error(c, "function %s documented but not found in code" % l[0])
306 except pp.ParseException, pe:
307 self.report_error(c, str(pe))
311 def report_error(self, c, msg):
312 print >>self.errors, "%s:%d: [%s] Error %s" % (c.filename, c.lineno, self.language, msg)
314 def cmd_begin(self, c):
315 if len(c.params) == 0:
316 self.report_error(c, "Malformed begin")
320 self.envstack.append((s, (c.filename, c.lineno)))
321 if s == 'description':
322 if self.language == 'py' and 'name' in self.function_props and not 'defpy' in self.function_props:
323 self.report_error(c, "No cvdefPy for function %s" % self.function_props['name'])
325 elif s == 'lstlisting':
326 # Set indent to zero while collecting code; so later write will not double-indent
327 self.saved_f = self.f
328 self.saved_indent = self.indent
329 self.f = StringIO.StringIO()
331 elif s in ['itemize', 'enumerate']:
334 self.f = StringIO.StringIO()
338 def cmd_item(self, c):
339 if len(self.ee()) == 0:
340 self.report_error(c, "item without environment")
343 markup = {'itemize' : '*', 'enumerate' : '#.', 'description' : '*'}[self.ee()[-1]]
345 markup += " " + self.render([c.args[0].str])
346 if len(c.params) > 0:
347 markup += " " + self.render(c.params[0].str)
348 self.write("\n\n" + markup)
351 def cmd_end(self, c):
352 if len(c.params) != 1:
353 self.report_error(c, "Malformed end")
355 if len(self.envstack) == 0:
356 self.report_error(c, "end with no env")
360 if self.envstack == []:
361 print "Cannot pop at", (c.filename, c.lineno)
362 if self.envstack[-1][0] != s:
363 self.report_error(c, "end{%s} does not match current stack %s" % (s, repr(self.envstack)))
365 if s == 'description':
368 self.function_props['done'] = True
369 elif s in ['itemize', 'enumerate']:
372 tabletxt = self.f.getvalue()
373 self.f = self.f_section
374 self.f.write(self.handle_table(tabletxt))
375 elif s == 'lstlisting':
376 listing = self.f.getvalue()
378 self.f = self.saved_f
379 self.indent = self.saved_indent
381 if self.language == 'py':
382 ckeys = ['#define', 'void', '#include', ';\n']
383 found = [repr(k) for k in ckeys if k in listing]
385 self.report_error(c, 'listing is probably C, found %s' % ",".join(found))
386 if (self.language == 'py') and ('>>>' in listing):
387 print >>self, "\n.. doctest::\n"
389 print >>self, "\n::\n"
396 print >>self, ".." # otherwise a following :param: gets treated as more listing
397 elif s == 'document':
402 def cmd_label(self, c):
405 def cmd_lstinputlisting(self, c):
407 print >>self.f, ".. include:: %s" % os.path.normpath(os.path.join(self.abspath, s))
408 print >>self.f, " :literal:"
412 def cmd_cvC(self, c):
413 self.do_conditional(['c'], c)
414 def cmd_cvCpp(self, c):
415 self.do_conditional(['cpp'], c)
416 def cmd_cvPy(self, c):
417 self.do_conditional(['py'], c)
418 def cmd_cvCPy(self, c):
419 self.do_conditional(['c', 'py'], c)
420 def do_conditional(self, langs, c):
421 if self.language in langs:
422 self.doL(c.params[0].str, False)
425 """ return L rendered as a string """
427 self.f = StringIO.StringIO()
429 if isinstance(x, TexCmd):
433 r = self.f.getvalue()
437 def cmd_cvarg(self, c):
438 if len(c.params) != 2:
439 self.report_error(c, "Malformed cvarg")
442 if self.state == 'class':
443 nm = self.render(c.params[0].str)
445 print >>self, "\n\n.. method:: %s\n\n" % nm
447 print >>self, "\n\n.. attribute:: %s\n\n" % nm
450 self.doL(c.params[1].str, False)
454 is_func_arg = (e == ['description']) and (not 'done' in self.function_props)
456 nm = self.render(c.params[0].str)
457 print >>self, '\n:param %s: ' % nm,
458 type = None # Try to figure out the argument type
459 # For now, multiple args get a pass
460 if (self.language == 'py') and ('signature' in self.function_props) and (not ',' in nm):
461 sig = self.function_props['signature']
462 argnames = [a[0] for a in sig[1]]
463 if isinstance(sig[2], str):
466 resnames = list(sig[2])
467 if not nm in argnames + resnames:
468 self.report_error(c, "Argument %s is not mentioned in signature (%s) (%s)" % (nm, ", ".join(argnames), ", ".join(resnames)))
470 api = python_api.get(self.function_props['name'], None)
473 adict = dict([(a.nm, a) for a in ins])
474 arg = adict.get(nm, None)
478 self.report_error(c, 'cannot find arg %s in code' % nm)
479 elif len(e) > 0 and e[-1] == 'description':
480 print >>self, '\n* **%s** ' % self.render(c.params[0].str),
482 self.report_error(c, "unexpected env (%s) for cvarg" % ",".join(e))
484 self.doL(c.params[1].str, False)
487 if is_func_arg and type:
488 type = type.replace('*', '')
490 "ints" : "sequence of int",
491 "floats" : "sequence of int",
492 "IplImages" : "sequence of :class:`IplImage`",
497 "cvarrseq" : ":class:`CvArr` or :class:`CvSeq`",
498 "CvPoint2D32fs" : "sequence of (float, float)",
499 "pts_npts_contours" : "list of lists of (x,y) pairs",
500 "CvSeqOfCvSURFPoint" : ":class:`CvSeq` of :class:`CvSURFPoint`",
501 "CvSeqOfCvSURFDescriptor" : ":class:`CvSeq` of list of float",
502 "cvpoint2d32f_count" : "int",
503 "ranges" : "list of tuples of ints",
504 "PyObject" : "object",
505 "edgeorpoint" : ":class:`CvSubdiv2DEdge`, :class:`CvSubdiv2DPoint`",
507 print >>self, "\n:type %s: %s" % (nm, translate.get(type, ':class:`%s`' % type))
509 def cmd_genc(self, c): pass
510 def cmd_genpy(self, c): pass
511 def cmd_author(self, c): pass
512 def cmd_date(self, c): pass
513 def cmd_def(self, c): pass
514 def cmd_documentclass(self, c): pass
515 def cmd_maketitle(self, c): pass
516 def cmd_newcommand(self, c): pass
517 def cmd_newline(self, c): pass
518 def cmd_setcounter(self, c): pass
519 def cmd_tableofcontents(self, c): pass
520 def cmd_targetlang(self, c): pass
521 def cmd_usepackage(self, c): pass
522 def cmd_title(self, c): pass
523 def cmd_par(self, c): pass
524 def cmd_hline(self, c):
525 print >>self, "\\hline"
527 def cmd_cite(self, c):
528 self.write("[%s]_" % str(c.params[0]))
530 def cmd_href(self, c):
531 if len(c.params) == 2:
532 self.write("`%s <%s>`_" % (str(c.params[1]), self.render(c.params[0].str)))
534 self.report_error(c, "href should have two params")
536 def cmd_url(self, c):
537 self.write(str(c.params[0]))
539 def cmd_emph(self, c):
540 self.write("*" + self.render(c.params[0].str) + "*")
542 def cmd_textit(self, c):
543 self.write("*" + self.render(c.params[0].str) + "*")
545 def cmd_textbf(self, c):
546 self.write("**" + self.render(c.params[0].str) + "**")
548 def cmd_texttt(self, c):
549 self.write("``" + self.render(c.params[0].str) + "``")
551 def default_cmd(self, c):
552 if self.f == self.f_section:
555 def unrecognized_cmd(self, c):
556 # if writing the index or chapter heading, anything goes
557 if not self.f in [self.f_index, self.f_chapter]:
559 if (not 'lstlisting' in self.ee()) and (not c.cmd in "#{}%&*\\_"):
560 if not c.cmd in self.unhandled_commands:
561 self.report_error(c, 'unhandled command %s' % c.cmd)
562 self.unhandled_commands.add(c.cmd)
564 def doL(self, L, newlines = True):
567 if isinstance(x, TexCmd):
570 if 'lstlisting' in self.ee() or not newlines:
573 self.doplain(x.lstrip())
576 if self.state in ['math'] or not newlines:
579 if not 'lstlisting' in self.ee():
582 def handle_table(self, s):
583 oneline = s.replace('\n', ' ').strip()
584 rows = [r.strip() for r in oneline.split('\\hline')]
588 cols = [c.strip() for c in r.split('&')]
590 widths = [max([len(r[i]) for r in tab]) for i in range(len(tab[0]))]
592 st = "" # Sphinx table
595 sep = "+" + "+".join(["-" * w for w in widths]) + "+"
598 st += "|" + "|".join([c.center(w) for (c, w) in zip(r, widths)]) + "|" + '\n'
601 st = '.. table::\n\n'
602 sep = " ".join(["=" * w for w in widths])
603 st += ' ' + sep + '\n'
604 for y,r in enumerate(tab):
605 st += ' ' + " ".join([c.ljust(w) for (c, w) in zip(r, widths)]) + '\n'
607 st += ' ' + sep + '\n'
608 st += ' ' + sep + '\n'
612 """ Return tags of the envstack. envstack[0] is 'document', so skip it """
613 return [n for (n,_) in self.envstack[1:]]
620 if self.envstack != []:
621 print >>self.errors, "Error envstack not empty at end of doc: " + repr(self.envstack)
622 print >>self.errors, "Unrecognized commands:"
623 for c in sorted(self.unhandled_commands):
624 print >>self.errors, "\n " + c
626 if self.language == 'py':
627 print >>self.errors, "The following functions are undocumented"
628 for f in sorted(set(python_api) - self.covered):
629 print >>self.errors, ' ', f
631 print >>self.f_index, " bibliography"
632 print >>self.f_index, """
641 # Quick and dirty bibtex parser
643 def parseBib(filename, language):
644 pp.ParserElement.setDefaultWhitespaceChars(" \n\t")
645 entry = returnList(pp.Word('@', pp.alphanums) + sl('{') +
646 pp.Word(pp.alphanums + "_") + sl(',') +
647 CommaList(returnTuple(pp.Word(pp.alphanums) + sl('=') + pp.QuotedString('{', endQuoteChar = '}'))) +
648 pp.Suppress(pp.Optional(',')) +
650 r = (pp.ZeroOrMore(entry) | pp.Suppress('#' + pp.ZeroOrMore(pp.CharsNotIn('\n'))) + pp.StringEnd()).parseFile(filename)
652 bibliography = QOpen(os.path.join(language, "bibliography.rst"), 'wt')
653 print >>bibliography, "Bibliography"
654 print >>bibliography, "============"
657 for _,e in sorted([(str(x[1]), x) for x in r]):
658 (etype, tag, attrs) = str(e[0][1:]), str(e[1]), dict([(str(a), str(b)) for (a,b) in e[2]])
661 'article' : '$author, "$title". $journal $volume $number, pp $pages ($year)',
662 'inproceedings' : '$author "$title", $booktitle, $year',
663 'misc' : '$author "$title", $year',
664 'techreport' : '$author "$title", $edition, $edition ($year)',
666 if etype in representations:
668 print >>bibliography, tag
669 print >>bibliography, "^" * len(tag)
672 print >>bibliography, ".. [%s] %s" % (tag, Template(representations[etype]).safe_substitute(attrs))
677 fulldoc = latexparser(sys.argv[1])
679 abspath = os.path.abspath(os.path.dirname(sys.argv[1]))
681 raw = open('raw.full', 'w')
686 # Filter on target language
687 def preprocess_conditionals(fd, conditionals):
691 if isinstance(x, TexCmd):
693 loc = (x.filename, x.lineno)
694 if ll.startswith("if"):
695 # print " " * len(ifstack), '{', loc
696 ifstack.append((conditionals.get(ll[2:], False), loc))
697 elif ll.startswith("else"):
698 ifstack[-1] = (not ifstack[-1][0], ifstack[-1][1])
699 elif ll.startswith("fi"):
701 # print " " * len(ifstack), '}', loc
702 elif not False in [p for (p,_) in ifstack]:
705 if not False in [p for (p,_) in ifstack]:
708 print "unterminated if", ifstack
713 for language in sys.argv[2:]:
714 doc = preprocess_conditionals(fulldoc, {
716 'Python' : language=='py',
717 'Py' : language=='py',
718 'CPy' : (language=='py' or language == 'c'),
719 'Cpp' : language=='cpp',
722 raw = open('raw.%s' % language, 'w')
726 sr = SphinxWriter('index.rst', language, abspath)
728 OpenCV |version| %s Reference
729 =================================
731 The OpenCV Wiki is here: http://opencv.willowgarage.com/
738 """ % {'c': 'C', 'cpp': 'C++', 'py': 'Python'}[language]
741 parseBib('../opencv.bib', language)
742 tags.update(sr.get_tags())
743 open('TAGS', 'w').write("\n".join(sorted(tags.values())) + "\n")