utils/demangle_tree.py

# Given a path to llvm-objdump and a directory tree, spider the directory tree
# dumping every object file encountered with correct options needed to demangle
# symbols in the object file, and collect statistics about failed / crashed
# demanglings.  Useful for stress testing the demangler against a large corpus
# of inputs.

from __future__ import print_function

import argparse
import functools
import os
import re
import sys
import subprocess
import traceback
from multiprocessing import Pool
import multiprocessing

args = None


def parse_line(line):
    question = line.find("?")
    if question == -1:
        return None, None

    open_paren = line.find("(", question)
    if open_paren == -1:
        return None, None
    close_paren = line.rfind(")", open_paren)
    if open_paren == -1:
        return None, None
    mangled = line[question:open_paren]
    demangled = line[open_paren + 1 : close_paren]
    return mangled.strip(), demangled.strip()


class Result(object):
    def __init__(self):
        self.crashed = []
        self.file = None
        self.nsymbols = 0
        self.errors = set()
        self.nfiles = 0


class MapContext(object):
    def __init__(self):
        self.rincomplete = None
        self.rcumulative = Result()
        self.pending_objs = []
        self.npending = 0


def process_file(path, objdump):
    r = Result()
    r.file = path

    popen_args = [objdump, "-t", "-demangle", path]
    p = subprocess.Popen(popen_args, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    stdout, stderr = p.communicate()
    if p.returncode != 0:
        r.crashed = [r.file]
        return r

    output = stdout.decode("utf-8")

    for line in output.splitlines():
        mangled, demangled = parse_line(line)
        if mangled is None:
            continue
        r.nsymbols += 1
        if "invalid mangled name" in demangled:
            r.errors.add(mangled)
    return r


def add_results(r1, r2):
    r1.crashed.extend(r2.crashed)
    r1.errors.update(r2.errors)
    r1.nsymbols += r2.nsymbols
    r1.nfiles += r2.nfiles


def print_result_row(directory, result):
    print(
        "[{0} files, {1} crashes, {2} errors, {3} symbols]: '{4}'".format(
            result.nfiles,
            len(result.crashed),
            len(result.errors),
            result.nsymbols,
            directory,
        )
    )


def process_one_chunk(pool, chunk_size, objdump, context):
    objs = []

    incomplete = False
    dir_results = {}
    ordered_dirs = []
    while context.npending > 0 and len(objs) < chunk_size:
        this_dir = context.pending_objs[0][0]
        ordered_dirs.append(this_dir)
        re = Result()
        if context.rincomplete is not None:
            re = context.rincomplete
            context.rincomplete = None

        dir_results[this_dir] = re
        re.file = this_dir

        nneeded = chunk_size - len(objs)
        objs_this_dir = context.pending_objs[0][1]
        navail = len(objs_this_dir)
        ntaken = min(nneeded, navail)
        objs.extend(objs_this_dir[0:ntaken])
        remaining_objs_this_dir = objs_this_dir[ntaken:]
        context.pending_objs[0] = (context.pending_objs[0][0], remaining_objs_this_dir)
        context.npending -= ntaken
        if ntaken == navail:
            context.pending_objs.pop(0)
        else:
            incomplete = True

        re.nfiles += ntaken

    assert len(objs) == chunk_size or context.npending == 0

    copier = functools.partial(process_file, objdump=objdump)
    mapped_results = list(pool.map(copier, objs))

    for mr in mapped_results:
        result_dir = os.path.dirname(mr.file)
        result_entry = dir_results[result_dir]
        add_results(result_entry, mr)

    # It's only possible that a single item is incomplete, and it has to be the
    # last item.
    if incomplete:
        context.rincomplete = dir_results[ordered_dirs[-1]]
        ordered_dirs.pop()

    # Now ordered_dirs contains a list of all directories which *did* complete.
    for c in ordered_dirs:
        re = dir_results[c]
        add_results(context.rcumulative, re)
        print_result_row(c, re)


def process_pending_files(pool, chunk_size, objdump, context):
    while context.npending >= chunk_size:
        process_one_chunk(pool, chunk_size, objdump, context)


def go():
    global args

    obj_dir = args.dir
    extensions = args.extensions.split(",")
    extensions = [x if x[0] == "." else "." + x for x in extensions]

    pool_size = 48
    pool = Pool(processes=pool_size)

    try:
        nfiles = 0
        context = MapContext()

        for root, dirs, files in os.walk(obj_dir):
            root = os.path.normpath(root)
            pending = []
            for f in files:
                file, ext = os.path.splitext(f)
                if not ext in extensions:
                    continue

                nfiles += 1
                full_path = os.path.join(root, f)
                full_path = os.path.normpath(full_path)
                pending.append(full_path)

            # If this directory had no object files, just print a default
            # status line and continue with the next dir
            if len(pending) == 0:
                print_result_row(root, Result())
                continue

            context.npending += len(pending)
            context.pending_objs.append((root, pending))
            # Drain the tasks, `pool_size` at a time, until we have less than
            # `pool_size` tasks remaining.
            process_pending_files(pool, pool_size, args.objdump, context)

        assert context.npending < pool_size
        process_one_chunk(pool, pool_size, args.objdump, context)

        total = context.rcumulative
        nfailed = len(total.errors)
        nsuccess = total.nsymbols - nfailed
        ncrashed = len(total.crashed)

        if nfailed > 0:
            print("Failures:")
            for m in sorted(total.errors):
                print("  " + m)
        if ncrashed > 0:
            print("Crashes:")
            for f in sorted(total.crashed):
                print("  " + f)
        print("Summary:")
        spct = float(nsuccess) / float(total.nsymbols)
        fpct = float(nfailed) / float(total.nsymbols)
        cpct = float(ncrashed) / float(nfiles)
        print("Processed {0} object files.".format(nfiles))
        print(
            "{0}/{1} symbols successfully demangled ({2:.4%})".format(
                nsuccess, total.nsymbols, spct
            )
        )
        print("{0} symbols could not be demangled ({1:.4%})".format(nfailed, fpct))
        print("{0} files crashed while demangling ({1:.4%})".format(ncrashed, cpct))

    except:
        traceback.print_exc()

    pool.close()
    pool.join()


if __name__ == "__main__":
    def_obj = "obj" if sys.platform == "win32" else "o"

    parser = argparse.ArgumentParser(
        description="Demangle all symbols in a tree of object files, looking for failures."
    )
    parser.add_argument(
        "dir", type=str, help="the root directory at which to start crawling"
    )
    parser.add_argument(
        "--objdump",
        type=str,
        help="path to llvm-objdump.  If not specified "
        + "the tool is located as if by `which llvm-objdump`.",
    )
    parser.add_argument(
        "--extensions",
        type=str,
        default=def_obj,
        help="comma separated list of extensions to demangle (e.g. `o,obj`).  "
        + "By default this will be `obj` on Windows and `o` otherwise.",
    )

    args = parser.parse_args()

    multiprocessing.freeze_support()
    go()