tor/scripts/maint/redox.py

#!/usr/bin/env python
#
#  Copyright (c) 2008-2019, The Tor Project, Inc.
#  See LICENSE for licensing information.
#
# Hi!
# I'm redox.py, the Tor redocumentation tool!
# I am a horrible hack!
# I read the output of doxygen from stderr, and add missing DOCDOC comments
#   to tell you where documentation should go!
# To use me, edit the stuff below...
#  ...and run 'make doxygen 2>doxygen.stderr' ...
#  ...and run ./scripts/maint/redox.py < doxygen.stderr !
# I'll make a bunch of new files by adding missing DOCDOC comments to your
#    source.  Those files will have names like ./src/common/util.c.newdoc.
# You will want to look over the changes by hand before checking them in.
#
# So, here's your workflow:
#
# 0. Make sure you're running a bourne shell for the redirects below.
# 1. make doxygen 1>doxygen.stdout 2>doxygen.stderr.
# 2. grep Warning doxygen.stderr | grep -v 'is not documented' | less
#      [This will tell you about all the bogus doxygen output you have]
# 3. python ./scripts/maint/redox.py <doxygen.stderr
#      [This will make lots of .newdoc files with DOCDOC comments for
#       whatever was missing documentation.]
# 4. Look over those .newdoc files, and see which docdoc comments you
#     want to merge into the main file.  If it's all good, just run
#     "mv fname.c.newdoc fname.c".  Otherwise, you'll need to merge
#     the parts you like by hand.

# Future imports for Python 2.7, mandatory in 3.0
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import re
import sys

try:
    xrange  # Python 2
except NameError:
    xrange = range  # Python 3

# Which files should we ignore warning from?  Mostly, these are external
# files that we've snarfed in from somebody else, whose C we do no intend
# to document for them.
SKIP_FILES = [ "OpenBSD_malloc_Linux.c",
               "strlcat.c",
               "strlcpy.c",
               "sha256.c",
               "sha256.h",
               "aes.c",
               "aes.h" ]

# What names of things never need javadoc
SKIP_NAME_PATTERNS = [ r'^.*_c_id$',
                       r'^.*_H_ID$' ]

# Which types of things should get DOCDOC comments added if they are
# missing documentation?  Recognized types are in KINDS below.
ADD_DOCDOCS_TO_TYPES = [ 'function', 'type', 'typedef' ]
ADD_DOCDOCS_TO_TYPES += [ 'variable', ]

# ====================
# The rest of this should not need hacking.

KINDS = [ "type", "field", "typedef", "define", "function", "variable",
          "enumeration" ]

NODOC_LINE_RE = re.compile(r'^([^:]+):(\d+): (\w+): (.*) is not documented\.$')

THING_RE = re.compile(r'^Member ([a-zA-Z0-9_]+).*\((typedef|define|function|variable|enumeration|macro definition)\) of (file|class) ')

SKIP_NAMES = [re.compile(s) for s in SKIP_NAME_PATTERNS]

def parsething(thing):
    """I figure out what 'foobar baz in quux quum is not documented' means,
       and return: the name of the foobar, and the kind of the foobar.
    """
    if thing.startswith("Compound "):
        tp, name = "type", thing.split()[1]
    else:
        m = THING_RE.match(thing)
        if not m:
            print(thing, "???? Format didn't match.")
            return None, None
        else:
            name, tp, parent = m.groups()
            if parent == 'class':
                if tp == 'variable' or tp == 'function':
                    tp = 'field'

    return name, tp

def read():
    """I snarf doxygen stderr from stdin, and parse all the "foo has no
       documentation messages.  I return a map from filename to lists
       of tuples of (alleged line number, name of thing, kind of thing)
    """
    errs = {}
    for line in sys.stdin:
        m = NODOC_LINE_RE.match(line)
        if m:
            file, line, tp, thing = m.groups()
            assert tp.lower() == 'warning'
            name, kind = parsething(thing)
            errs.setdefault(file, []).append((int(line), name, kind))

    return errs

def findline(lines, lineno, ident):
    """Given a list of all the lines in the file (adjusted so 1-indexing works),
       a line number that ident is allegedly on, and ident, I figure out
       the line where ident was really declared."""
    lno = lineno
    for lineno in xrange(lineno, 0, -1):
        try:
            if ident in lines[lineno]:
                return lineno
        except IndexError:
            continue

    return None

FUNC_PAT = re.compile(r"^[A-Za-z0-9_]+\(")

def hascomment(lines, lineno, kind):
    """I return true if it looks like there's already a good comment about
       the thing on lineno of lines of type kind. """
    if "*/" in lines[lineno-1]:
        return True
    if kind == 'function' and FUNC_PAT.match(lines[lineno]):
        if "*/" in lines[lineno-2]:
            return True
    return False

def hasdocdoc(lines, lineno, kind):
    """I return true if it looks like there's already a docdoc comment about
       the thing on lineno of lines of type kind."""
    try:
        if "DOCDOC" in lines[lineno]:
            return True
    except IndexError:
        pass
    try:
        if "DOCDOC" in lines[lineno-1]:
            return True
    except IndexError:
        pass
    if kind == 'function' and FUNC_PAT.match(lines[lineno]):
        if "DOCDOC" in lines[lineno-2]:
            return True
    return False

def checkf(fn, errs):
    """I go through the output of read() for a single file, and build a list
       of tuples of things that want DOCDOC comments.  Each tuple has:
       the line number where the comment goes; the kind of thing; its name.
    """
    for skip in SKIP_FILES:
        if fn.endswith(skip):
            print("Skipping",fn)
            return

    comments = []
    lines = [ None ]
    try:
        lines.extend( open(fn, 'r').readlines() )
    except IOError:
        return

    for line, name, kind in errs:
        if any(pat.match(name) for pat in SKIP_NAMES):
            continue

        if kind not in ADD_DOCDOCS_TO_TYPES:
            continue

        ln = findline(lines, line, name)
        if ln == None:
            print("Couldn't find the definition of %s allegedly on %s of %s"%(
                name, line, fn))
        else:
            if hasdocdoc(lines, line, kind):
#                print "Has a DOCDOC"
#                print fn, line, name, kind
#                print "\t",lines[line-2],
#                print "\t",lines[line-1],
#                print "\t",lines[line],
#                print "-------"
                pass
            else:
                if kind == 'function' and FUNC_PAT.match(lines[ln]):
                    ln = ln - 1

                comments.append((ln, kind, name))

    return comments

def applyComments(fn, entries):
    """I apply lots of comments to the file in fn, making a new .newdoc file.
    """
    N = 0

    lines = [ None ]
    try:
        lines.extend( open(fn, 'r').readlines() )
    except IOError:
        return

    # Process the comments in reverse order by line number, so that
    # the line numbers for the ones we haven't added yet remain valid
    # until we add them.  Standard trick.
    entries.sort()
    entries.reverse()

    for ln, kind, name in entries:

        lines.insert(ln, "/* DOCDOC %s */\n"%name)
        N += 1

    outf = open(fn+".newdoc", 'w')
    for line in lines[1:]:
        outf.write(line)
    outf.close()

    print("Added %s DOCDOCs to %s" %(N, fn))

e = read()

for fn, errs in e.iteritems():
    print(repr((fn, errs)))
    comments = checkf(fn, errs)
    if comments:
        applyComments(fn, comments)