utils/extract_symbols.py - llvm-project/llvm - Git at Google

 #!/usr/bin/env python

 """A tool for extracting a list of symbols to export

 When exporting symbols from a dll or exe we either need to mark the symbols in
 the source code as __declspec(dllexport) or supply a list of symbols to the
 linker. This program automates the latter by inspecting the symbol tables of a
 list of link inputs and deciding which of those symbols need to be exported.

 We can't just export all the defined symbols, as there's a limit of 65535
 exported symbols and in clang we go way over that, particularly in a debug
 build. Therefore a large part of the work is pruning symbols either which can't
 be imported, or which we think are things that have definitions in public header
 files (i.e. template instantiations) and we would get defined in the thing
 importing these symbols anyway.
 """

 from __future__ import print_function
 import sys
 import re
 import os
 import subprocess
 import multiprocessing
 import argparse
 import platform

 # Define a function which extracts a list of pairs of (symbols, is_def) from a
 # library using llvm-nm becuase it can work both with regular and bitcode files.
 # We use subprocess.Popen and yield a symbol at a time instead of using
 # subprocess.check_output and returning a list as, especially on Windows, waiting
 # for the entire output to be ready can take a significant amount of time.
 def nm_get_symbols(tool, lib):
     # '-P' means the output is in portable format,
     # '-g' means we only get global symbols,
     # '-Xany' enforce handling both 32- and 64-bit objects on AIX,
     # '--no-demangle' ensure that C++ symbol names are not demangled; note
     #   that llvm-nm do not demangle by default, but the system nm on AIX does
     #   that, so the behavior may change in the future,
     # '-p' do not waste time sorting the symbols.
     cmd = [tool, "-P", "-g", "-Xany", "--no-demangle", "-p"]
     process = subprocess.Popen(
         cmd + [lib],
         bufsize=1,
         stdout=subprocess.PIPE,
         stdin=subprocess.PIPE,
         universal_newlines=True,
     )
     process.stdin.close()
     for line in process.stdout:
         # Look for external symbols that are defined in some section
         # The POSIX format is:
         #   name   type   value   size
         # The -P flag displays the size field for symbols only when applicable,
         # so the last field is optional. There's no space after the value field,
         # but \s+ match newline also, so \s+\S* will match the optional size field.
         match = re.match("^(\S+)\s+[BDGRSTuVW]\s+\S+\s+\S*$", line)
         if match:
             yield (match.group(1), True)
         # Look for undefined symbols, which have type U and may or may not
         # (depending on which nm is being used) have value and size.
         match = re.match("^(\S+)\s+U\s+(\S+\s+\S*)?$", line)
         if match:
             yield (match.group(1), False)
     process.wait()


 # Define a function which determines if the target is 32-bit Windows (as that's
 # where calling convention name decoration happens).
 def readobj_is_32bit_windows(tool, lib):
     output = subprocess.check_output(
         [tool, "--file-header", lib], universal_newlines=True
     )
     for line in output.splitlines():
         match = re.match("Format: (\S+)", line)
         if match:
             return match.group(1) == "COFF-i386"
     return False


 # MSVC mangles names to ?<identifier_mangling>@<type_mangling>. By examining the
 # identifier/type mangling we can decide which symbols could possibly be
 # required and which we can discard.
 def should_keep_microsoft_symbol(symbol, calling_convention_decoration):
     # Keep unmangled (i.e. extern "C") names
     if not "?" in symbol:
         if calling_convention_decoration:
             # Remove calling convention decoration from names
             match = re.match("[_@]([^@]+)", symbol)
             if match:
                 symbol = match.group(1)
         # Discard floating point/SIMD constants.
         if symbol.startswith(("__xmm@", "__ymm@", "__real@")):
             return None
         return symbol
     # Deleting destructors start with ?_G or ?_E and can be discarded because
     # link.exe gives you a warning telling you they can't be exported if you
     # don't
     elif symbol.startswith("??_G") or symbol.startswith("??_E"):
         return None
     # An anonymous namespace is mangled as ?A(maybe hex number)@. Any symbol
     # that mentions an anonymous namespace can be discarded, as the anonymous
     # namespace doesn't exist outside of that translation unit.
     elif re.search("\?A(0x\w+)?@", symbol):
         return None
     # Skip X86GenMnemonicTables functions, they are not exposed from llvm/include/.
     elif re.match("\?is[A-Z0-9]*@X86@llvm", symbol):
         return None
     # Keep mangled llvm:: and clang:: function symbols. How we detect these is a
     # bit of a mess and imprecise, but that avoids having to completely demangle
     # the symbol name. The outermost namespace is at the end of the identifier
     # mangling, and the identifier mangling is followed by the type mangling, so
     # we look for (llvm|clang)@@ followed by something that looks like a
     # function type mangling. To spot a function type we use (this is derived
     # from clang/lib/AST/MicrosoftMangle.cpp):
     # <function-type> ::= <function-class> <this-cvr-qualifiers>
     #                     <calling-convention> <return-type>
     #                     <argument-list> <throw-spec>
     # <function-class> ::= [A-Z]
     # <this-cvr-qualifiers> ::= [A-Z0-9_]*
     # <calling-convention> ::= [A-JQ]
     # <return-type> ::= .+
     # <argument-list> ::= X   (void)
     #                 ::= .+@ (list of types)
     #                 ::= .*Z (list of types, varargs)
     # <throw-spec> ::= exceptions are not allowed
     elif re.search("(llvm|clang)@@[A-Z][A-Z0-9_]*[A-JQ].+(X|.+@|.*Z)$", symbol):
         return symbol
     return None


 # Itanium manglings are of the form _Z<identifier_mangling><type_mangling>. We
 # demangle the identifier mangling to identify symbols that can be safely
 # discarded.
 def should_keep_itanium_symbol(symbol, calling_convention_decoration):
     # Start by removing any calling convention decoration (which we expect to
     # see on all symbols, even mangled C++ symbols)
     if calling_convention_decoration and symbol.startswith("_"):
         symbol = symbol[1:]
     # Keep unmangled names
     if not symbol.startswith("_") and not symbol.startswith("."):
         return symbol
     # Discard manglings that aren't nested names
     match = re.match("_Z(T[VTIS])?(N.+)", symbol)
     if not match:
         return None
     # Demangle the name. If the name is too complex then we don't need to keep
     # it, but it the demangling fails then keep the symbol just in case.
     try:
         names, _ = parse_itanium_nested_name(match.group(2))
     except TooComplexName:
         return None
     if not names:
         return symbol
     # Keep llvm:: and clang:: names
     elif names[0][0] == "4llvm" or names[0][0] == "5clang":
         return symbol
     # Discard everything else
     else:
         return None


 # Certain kinds of complex manglings we assume cannot be part of a public
 # interface, and we handle them by raising an exception.
 class TooComplexName(Exception):
     pass


 # Parse an itanium mangled name from the start of a string and return a
 # (name, rest of string) pair.
 def parse_itanium_name(arg):
     # Check for a normal name
     match = re.match("(\d+)(.+)", arg)
     if match:
         n = int(match.group(1))
         name = match.group(1) + match.group(2)[:n]
         rest = match.group(2)[n:]
         return name, rest
     # Check for constructor/destructor names
     match = re.match("([CD][123])(.+)", arg)
     if match:
         return match.group(1), match.group(2)
     # Assume that a sequence of characters that doesn't end a nesting is an
     # operator (this is very imprecise, but appears to be good enough)
     match = re.match("([^E]+)(.+)", arg)
     if match:
         return match.group(1), match.group(2)
     # Anything else: we can't handle it
     return None, arg


 # Parse an itanium mangled template argument list from the start of a string
 # and throw it away, returning the rest of the string.
 def skip_itanium_template(arg):
     # A template argument list starts with I
     assert arg.startswith("I"), arg
     tmp = arg[1:]
     while tmp:
         # Check for names
         match = re.match("(\d+)(.+)", tmp)
         if match:
             n = int(match.group(1))
             tmp = match.group(2)[n:]
             continue
         # Check for substitutions
         match = re.match("S[A-Z0-9]*_(.+)", tmp)
         if match:
             tmp = match.group(1)
         # Start of a template
         elif tmp.startswith("I"):
             tmp = skip_itanium_template(tmp)
         # Start of a nested name
         elif tmp.startswith("N"):
             _, tmp = parse_itanium_nested_name(tmp)
         # Start of an expression: assume that it's too complicated
         elif tmp.startswith("L") or tmp.startswith("X"):
             raise TooComplexName
         # End of the template
         elif tmp.startswith("E"):
             return tmp[1:]
         # Something else: probably a type, skip it
         else:
             tmp = tmp[1:]
     return None


 # Parse an itanium mangled nested name and transform it into a list of pairs of
 # (name, is_template), returning (list, rest of string).
 def parse_itanium_nested_name(arg):
     # A nested name starts with N
     assert arg.startswith("N"), arg
     ret = []

     # Skip past the N, and possibly a substitution
     match = re.match("NS[A-Z0-9]*_(.+)", arg)
     if match:
         tmp = match.group(1)
     else:
         tmp = arg[1:]

     # Skip past CV-qualifiers and ref qualifiers
     match = re.match("[rVKRO]*(.+)", tmp)
     if match:
         tmp = match.group(1)

     # Repeatedly parse names from the string until we reach the end of the
     # nested name
     while tmp:
         # An E ends the nested name
         if tmp.startswith("E"):
             return ret, tmp[1:]
         # Parse a name
         name_part, tmp = parse_itanium_name(tmp)
         if not name_part:
             # If we failed then we don't know how to demangle this
             return None, None
         is_template = False
         # If this name is a template record that, then skip the template
         # arguments
         if tmp.startswith("I"):
             tmp = skip_itanium_template(tmp)
             is_template = True
         # Add the name to the list
         ret.append((name_part, is_template))

     # If we get here then something went wrong
     return None, None


 # Parse a microsoft mangled symbol and return a list of pairs of
 # (name, is_template). This is very rudimentary and does just enough
 # in order to determine if the first or second component is a template.
 def parse_microsoft_mangling(arg):
     # If the name doesn't start with ? this isn't a mangled name
     if not arg.startswith("?"):
         return [(arg, False)]
     arg = arg[1:]
     components = []
     while len(arg) > 0:
         # If we see an empty component we've reached the end
         if arg.startswith("@"):
             return components
         # Check for a simple name
         match = re.match("(\w+)@(.+)", arg)
         if match:
             components.append((match.group(1), False))
             arg = match.group(2)
             continue
         # Check for a special function name
         match = re.match("(\?_?\w)(.+)", arg)
         if match:
             components.append((match.group(1), False))
             arg = match.group(2)
             continue
         # Check for a template name
         match = re.match("\?\$(\w+)@[^@]+@(.+)", arg)
         if match:
             components.append((match.group(1), True))
             arg = match.group(2)
             continue
         # Some other kind of name that we can't handle
         components.append((arg, False))
         return components
     return components


 def extract_symbols(arg):
     llvm_nm_path, should_keep_symbol, calling_convention_decoration, lib = arg
     symbol_defs = dict()
     symbol_refs = set()
     for (symbol, is_def) in nm_get_symbols(llvm_nm_path, lib):
         symbol = should_keep_symbol(symbol, calling_convention_decoration)
         if symbol:
             if is_def:
                 symbol_defs[symbol] = 1 + symbol_defs.setdefault(symbol, 0)
             else:
                 symbol_refs.add(symbol)
     return (symbol_defs, symbol_refs)


 def get_template_name(sym, mangling):
     # Parse the mangling into a list of (name, is_template)
     try:
         if mangling == "microsoft":
             names = parse_microsoft_mangling(sym)
         else:
             match = re.match("_Z(T[VTIS])?(N.+)", sym)
             if match:
                 names, _ = parse_itanium_nested_name(match.group(2))
             else:
                 names = None
     except TooComplexName:
         return None

     if not names:
         return None

     # If any component is a template then return it
     for name, is_template in names:
         if is_template:
             return name

     # Not a template
     return None


 def parse_tool_path(parser, tool, val):
     try:
         # Close std streams as we don't want any output and we don't
         # want the process to wait for something on stdin.
         p = subprocess.Popen(
             [val],
             stdout=subprocess.PIPE,
             stderr=subprocess.PIPE,
             stdin=subprocess.PIPE,
             universal_newlines=True,
         )
         p.stdout.close()
         p.stderr.close()
         p.stdin.close()
         p.wait()
         return val
     except Exception:
         parser.error(f"Invalid path for {tool}")


 if __name__ == "__main__":
     parser = argparse.ArgumentParser(
         description="Extract symbols to export from libraries"
     )
     parser.add_argument(
         "--mangling",
         choices=["itanium", "microsoft"],
         required=True,
         help="expected symbol mangling scheme",
     )
     parser.add_argument(
         "--nm",
         metavar="path",
         type=lambda x: parse_tool_path(parser, "nm", x),
         help="path to the llvm-nm executable",
     )
     parser.add_argument(
         "--readobj",
         metavar="path",
         type=lambda x: parse_tool_path(parser, "readobj", x),
         help="path to the llvm-readobj executable",
     )
     parser.add_argument(
         "libs",
         metavar="lib",
         type=str,
         nargs="+",
         help="libraries to extract symbols from",
     )
     parser.add_argument("-o", metavar="file", type=str, help="output to file")
     args = parser.parse_args()

     # How we determine which symbols to keep and which to discard depends on
     # the mangling scheme
     if args.mangling == "microsoft":
         should_keep_symbol = should_keep_microsoft_symbol
     else:
         should_keep_symbol = should_keep_itanium_symbol

     # Get the list of libraries to extract symbols from
     libs = list()
     for lib in args.libs:
         # When invoked by cmake the arguments are the cmake target names of the
         # libraries, so we need to add .lib/.a to the end and maybe lib to the
         # start to get the filename. Also allow objects.
         suffixes = [".lib", ".a", ".obj", ".o"]
         if not any([lib.endswith(s) for s in suffixes]):
             for s in suffixes:
                 if os.path.exists(lib + s):
                     lib = lib + s
                     break
                 if os.path.exists("lib" + lib + s):
                     lib = "lib" + lib + s
                     break
         if not any([lib.endswith(s) for s in suffixes]):
             print("Don't know what to do with argument " + lib, file=sys.stderr)
             exit(1)
         libs.append(lib)

     # Check if calling convention decoration is used by inspecting the first
     # library in the list
     calling_convention_decoration = readobj_is_32bit_windows(args.readobj, libs[0])

     # Extract symbols from libraries in parallel. This is a huge time saver when
     # doing a debug build, as there are hundreds of thousands of symbols in each
     # library.
     # FIXME: On AIX, the default pool size can be too big for a logical
     #        partition's allocated memory, and can lead to an out of memory
     #        IO error. We are setting the pool size to 8 to avoid such
     #        errors at the moment, and will look for a graceful solution later.
     pool = multiprocessing.Pool(8) if platform.system() == "AIX" \
                                    else multiprocessing.Pool()
     try:
         # Only one argument can be passed to the mapping function, and we can't
         # use a lambda or local function definition as that doesn't work on
         # windows, so create a list of tuples which duplicates the arguments
         # that are the same in all calls.
         vals = [
             (args.nm, should_keep_symbol, calling_convention_decoration, x)
             for x in libs
         ]
         # Do an async map then wait for the result to make sure that
         # KeyboardInterrupt gets caught correctly (see
         # http://bugs.python.org/issue8296)
         result = pool.map_async(extract_symbols, vals)
         pool.close()
         libs_symbols = result.get(3600)
     except KeyboardInterrupt:
         # On Ctrl-C terminate everything and exit
         pool.terminate()
         pool.join()
         exit(1)

     # Merge everything into a single dict
     symbol_defs = dict()
     symbol_refs = set()
     for (this_lib_defs, this_lib_refs) in libs_symbols:
         for k, v in list(this_lib_defs.items()):
             symbol_defs[k] = v + symbol_defs.setdefault(k, 0)
         for sym in list(this_lib_refs):
             symbol_refs.add(sym)

     # Find which template instantiations are referenced at least once.
     template_instantiation_refs = set()
     for sym in list(symbol_refs):
         template = get_template_name(sym, args.mangling)
         if template:
             template_instantiation_refs.add(template)

     # Print symbols which both:
     #  * Appear in exactly one input, as symbols defined in multiple
     #    objects/libraries are assumed to have public definitions.
     #  * Are not a template instantiation that isn't referenced anywhere. This
     #    is because we need to export any explicitly instantiated templates,
     #    and we expect those to be referenced in some object.
     if args.o:
         outfile = open(args.o, "w")
     else:
         outfile = sys.stdout
     for k, v in list(symbol_defs.items()):
         template = get_template_name(k, args.mangling)
         if v == 1 and (not template or template in template_instantiation_refs):
             print(k, file=outfile)
	#!/usr/bin/env python

	"""A tool for extracting a list of symbols to export

	When exporting symbols from a dll or exe we either need to mark the symbols in
	the source code as __declspec(dllexport) or supply a list of symbols to the
	linker. This program automates the latter by inspecting the symbol tables of a
	list of link inputs and deciding which of those symbols need to be exported.

	We can't just export all the defined symbols, as there's a limit of 65535
	exported symbols and in clang we go way over that, particularly in a debug
	build. Therefore a large part of the work is pruning symbols either which can't
	be imported, or which we think are things that have definitions in public header
	files (i.e. template instantiations) and we would get defined in the thing
	importing these symbols anyway.
	"""

	from __future__ import print_function
	import sys
	import re
	import os
	import subprocess
	import multiprocessing
	import argparse
	import platform

	# Define a function which extracts a list of pairs of (symbols, is_def) from a
	# library using llvm-nm becuase it can work both with regular and bitcode files.
	# We use subprocess.Popen and yield a symbol at a time instead of using
	# subprocess.check_output and returning a list as, especially on Windows, waiting
	# for the entire output to be ready can take a significant amount of time.
	def nm_get_symbols(tool, lib):
	# '-P' means the output is in portable format,
	# '-g' means we only get global symbols,
	# '-Xany' enforce handling both 32- and 64-bit objects on AIX,
	# '--no-demangle' ensure that C++ symbol names are not demangled; note
	# that llvm-nm do not demangle by default, but the system nm on AIX does
	# that, so the behavior may change in the future,
	# '-p' do not waste time sorting the symbols.
	cmd = [tool, "-P", "-g", "-Xany", "--no-demangle", "-p"]
	process = subprocess.Popen(
	cmd + [lib],
	bufsize=1,
	stdout=subprocess.PIPE,
	stdin=subprocess.PIPE,
	universal_newlines=True,
	)
	process.stdin.close()
	for line in process.stdout:
	# Look for external symbols that are defined in some section
	# The POSIX format is:
	# name type value size
	# The -P flag displays the size field for symbols only when applicable,
	# so the last field is optional. There's no space after the value field,
	# but \s+ match newline also, so \s+\S* will match the optional size field.
	match = re.match("^(\S+)\s+[BDGRSTuVW]\s+\S+\s+\S*$", line)
	if match:
	yield (match.group(1), True)
	# Look for undefined symbols, which have type U and may or may not
	# (depending on which nm is being used) have value and size.
	match = re.match("^(\S+)\s+U\s+(\S+\s+\S*)?$", line)
	if match:
	yield (match.group(1), False)
	process.wait()


	# Define a function which determines if the target is 32-bit Windows (as that's
	# where calling convention name decoration happens).
	def readobj_is_32bit_windows(tool, lib):
	output = subprocess.check_output(
	[tool, "--file-header", lib], universal_newlines=True
	)
	for line in output.splitlines():
	match = re.match("Format: (\S+)", line)
	if match:
	return match.group(1) == "COFF-i386"
	return False


	# MSVC mangles names to ?<identifier_mangling>@<type_mangling>. By examining the
	# identifier/type mangling we can decide which symbols could possibly be
	# required and which we can discard.
	def should_keep_microsoft_symbol(symbol, calling_convention_decoration):
	# Keep unmangled (i.e. extern "C") names
	if not "?" in symbol:
	if calling_convention_decoration:
	# Remove calling convention decoration from names
	match = re.match("[_@]([^@]+)", symbol)
	if match:
	symbol = match.group(1)
	# Discard floating point/SIMD constants.
	if symbol.startswith(("__xmm@", "__ymm@", "__real@")):
	return None
	return symbol
	# Deleting destructors start with ?_G or ?_E and can be discarded because
	# link.exe gives you a warning telling you they can't be exported if you
	# don't
	elif symbol.startswith("??_G") or symbol.startswith("??_E"):
	return None
	# An anonymous namespace is mangled as ?A(maybe hex number)@. Any symbol
	# that mentions an anonymous namespace can be discarded, as the anonymous
	# namespace doesn't exist outside of that translation unit.
	elif re.search("\?A(0x\w+)?@", symbol):
	return None
	# Skip X86GenMnemonicTables functions, they are not exposed from llvm/include/.
	elif re.match("\?is[A-Z0-9]*@X86@llvm", symbol):
	return None
	# Keep mangled llvm:: and clang:: function symbols. How we detect these is a
	# bit of a mess and imprecise, but that avoids having to completely demangle
	# the symbol name. The outermost namespace is at the end of the identifier
	# mangling, and the identifier mangling is followed by the type mangling, so
	# we look for (llvm\|clang)@@ followed by something that looks like a
	# function type mangling. To spot a function type we use (this is derived
	# from clang/lib/AST/MicrosoftMangle.cpp):
	# <function-type> ::= <function-class> <this-cvr-qualifiers>
	# <calling-convention> <return-type>
	# <argument-list> <throw-spec>
	# <function-class> ::= [A-Z]
	# <this-cvr-qualifiers> ::= [A-Z0-9_]*
	# <calling-convention> ::= [A-JQ]
	# <return-type> ::= .+
	# <argument-list> ::= X (void)
	# ::= .+@ (list of types)
	# ::= .*Z (list of types, varargs)
	# <throw-spec> ::= exceptions are not allowed
	elif re.search("(llvm\|clang)@@[A-Z][A-Z0-9_][A-JQ].+(X\|.+@\|.Z)$", symbol):
	return symbol
	return None


	# Itanium manglings are of the form _Z<identifier_mangling><type_mangling>. We
	# demangle the identifier mangling to identify symbols that can be safely
	# discarded.
	def should_keep_itanium_symbol(symbol, calling_convention_decoration):
	# Start by removing any calling convention decoration (which we expect to
	# see on all symbols, even mangled C++ symbols)
	if calling_convention_decoration and symbol.startswith("_"):
	symbol = symbol[1:]
	# Keep unmangled names
	if not symbol.startswith("_") and not symbol.startswith("."):
	return symbol
	# Discard manglings that aren't nested names
	match = re.match("_Z(T[VTIS])?(N.+)", symbol)
	if not match:
	return None
	# Demangle the name. If the name is too complex then we don't need to keep
	# it, but it the demangling fails then keep the symbol just in case.
	try:
	names, _ = parse_itanium_nested_name(match.group(2))
	except TooComplexName:
	return None
	if not names:
	return symbol
	# Keep llvm:: and clang:: names
	elif names[0][0] == "4llvm" or names[0][0] == "5clang":
	return symbol
	# Discard everything else
	else:
	return None


	# Certain kinds of complex manglings we assume cannot be part of a public
	# interface, and we handle them by raising an exception.
	class TooComplexName(Exception):
	pass


	# Parse an itanium mangled name from the start of a string and return a
	# (name, rest of string) pair.
	def parse_itanium_name(arg):
	# Check for a normal name
	match = re.match("(\d+)(.+)", arg)
	if match:
	n = int(match.group(1))
	name = match.group(1) + match.group(2)[:n]
	rest = match.group(2)[n:]
	return name, rest
	# Check for constructor/destructor names
	match = re.match("([CD][123])(.+)", arg)
	if match:
	return match.group(1), match.group(2)
	# Assume that a sequence of characters that doesn't end a nesting is an
	# operator (this is very imprecise, but appears to be good enough)
	match = re.match("([^E]+)(.+)", arg)
	if match:
	return match.group(1), match.group(2)
	# Anything else: we can't handle it
	return None, arg


	# Parse an itanium mangled template argument list from the start of a string
	# and throw it away, returning the rest of the string.
	def skip_itanium_template(arg):
	# A template argument list starts with I
	assert arg.startswith("I"), arg
	tmp = arg[1:]
	while tmp:
	# Check for names
	match = re.match("(\d+)(.+)", tmp)
	if match:
	n = int(match.group(1))
	tmp = match.group(2)[n:]
	continue
	# Check for substitutions
	match = re.match("S[A-Z0-9]*_(.+)", tmp)
	if match:
	tmp = match.group(1)
	# Start of a template
	elif tmp.startswith("I"):
	tmp = skip_itanium_template(tmp)
	# Start of a nested name
	elif tmp.startswith("N"):
	_, tmp = parse_itanium_nested_name(tmp)
	# Start of an expression: assume that it's too complicated
	elif tmp.startswith("L") or tmp.startswith("X"):
	raise TooComplexName
	# End of the template
	elif tmp.startswith("E"):
	return tmp[1:]
	# Something else: probably a type, skip it
	else:
	tmp = tmp[1:]
	return None


	# Parse an itanium mangled nested name and transform it into a list of pairs of
	# (name, is_template), returning (list, rest of string).
	def parse_itanium_nested_name(arg):
	# A nested name starts with N
	assert arg.startswith("N"), arg
	ret = []

	# Skip past the N, and possibly a substitution
	match = re.match("NS[A-Z0-9]*_(.+)", arg)
	if match:
	tmp = match.group(1)
	else:
	tmp = arg[1:]

	# Skip past CV-qualifiers and ref qualifiers
	match = re.match("[rVKRO]*(.+)", tmp)
	if match:
	tmp = match.group(1)

	# Repeatedly parse names from the string until we reach the end of the
	# nested name
	while tmp:
	# An E ends the nested name
	if tmp.startswith("E"):
	return ret, tmp[1:]
	# Parse a name
	name_part, tmp = parse_itanium_name(tmp)
	if not name_part:
	# If we failed then we don't know how to demangle this
	return None, None
	is_template = False
	# If this name is a template record that, then skip the template
	# arguments
	if tmp.startswith("I"):
	tmp = skip_itanium_template(tmp)
	is_template = True
	# Add the name to the list
	ret.append((name_part, is_template))

	# If we get here then something went wrong
	return None, None


	# Parse a microsoft mangled symbol and return a list of pairs of
	# (name, is_template). This is very rudimentary and does just enough
	# in order to determine if the first or second component is a template.
	def parse_microsoft_mangling(arg):
	# If the name doesn't start with ? this isn't a mangled name
	if not arg.startswith("?"):
	return [(arg, False)]
	arg = arg[1:]
	components = []
	while len(arg) > 0:
	# If we see an empty component we've reached the end
	if arg.startswith("@"):
	return components
	# Check for a simple name
	match = re.match("(\w+)@(.+)", arg)
	if match:
	components.append((match.group(1), False))
	arg = match.group(2)
	continue
	# Check for a special function name
	match = re.match("(\?_?\w)(.+)", arg)
	if match:
	components.append((match.group(1), False))
	arg = match.group(2)
	continue
	# Check for a template name
	match = re.match("\?\$(\w+)@[^@]+@(.+)", arg)
	if match:
	components.append((match.group(1), True))
	arg = match.group(2)
	continue
	# Some other kind of name that we can't handle
	components.append((arg, False))
	return components
	return components


	def extract_symbols(arg):
	llvm_nm_path, should_keep_symbol, calling_convention_decoration, lib = arg
	symbol_defs = dict()
	symbol_refs = set()
	for (symbol, is_def) in nm_get_symbols(llvm_nm_path, lib):
	symbol = should_keep_symbol(symbol, calling_convention_decoration)
	if symbol:
	if is_def:
	symbol_defs[symbol] = 1 + symbol_defs.setdefault(symbol, 0)
	else:
	symbol_refs.add(symbol)
	return (symbol_defs, symbol_refs)


	def get_template_name(sym, mangling):
	# Parse the mangling into a list of (name, is_template)
	try:
	if mangling == "microsoft":
	names = parse_microsoft_mangling(sym)
	else:
	match = re.match("_Z(T[VTIS])?(N.+)", sym)
	if match:
	names, _ = parse_itanium_nested_name(match.group(2))
	else:
	names = None
	except TooComplexName:
	return None

	if not names:
	return None

	# If any component is a template then return it
	for name, is_template in names:
	if is_template:
	return name

	# Not a template
	return None


	def parse_tool_path(parser, tool, val):
	try:
	# Close std streams as we don't want any output and we don't
	# want the process to wait for something on stdin.
	p = subprocess.Popen(
	[val],
	stdout=subprocess.PIPE,
	stderr=subprocess.PIPE,
	stdin=subprocess.PIPE,
	universal_newlines=True,
	)
	p.stdout.close()
	p.stderr.close()
	p.stdin.close()
	p.wait()
	return val
	except Exception:
	parser.error(f"Invalid path for {tool}")


	if __name__ == "__main__":
	parser = argparse.ArgumentParser(
	description="Extract symbols to export from libraries"
	)
	parser.add_argument(
	"--mangling",
	choices=["itanium", "microsoft"],
	required=True,
	help="expected symbol mangling scheme",
	)
	parser.add_argument(
	"--nm",
	metavar="path",
	type=lambda x: parse_tool_path(parser, "nm", x),
	help="path to the llvm-nm executable",
	)
	parser.add_argument(
	"--readobj",
	metavar="path",
	type=lambda x: parse_tool_path(parser, "readobj", x),
	help="path to the llvm-readobj executable",
	)
	parser.add_argument(
	"libs",
	metavar="lib",
	type=str,
	nargs="+",
	help="libraries to extract symbols from",
	)
	parser.add_argument("-o", metavar="file", type=str, help="output to file")
	args = parser.parse_args()

	# How we determine which symbols to keep and which to discard depends on
	# the mangling scheme
	if args.mangling == "microsoft":
	should_keep_symbol = should_keep_microsoft_symbol
	else:
	should_keep_symbol = should_keep_itanium_symbol

	# Get the list of libraries to extract symbols from
	libs = list()
	for lib in args.libs:
	# When invoked by cmake the arguments are the cmake target names of the
	# libraries, so we need to add .lib/.a to the end and maybe lib to the
	# start to get the filename. Also allow objects.
	suffixes = [".lib", ".a", ".obj", ".o"]
	if not any([lib.endswith(s) for s in suffixes]):
	for s in suffixes:
	if os.path.exists(lib + s):
	lib = lib + s
	break
	if os.path.exists("lib" + lib + s):
	lib = "lib" + lib + s
	break
	if not any([lib.endswith(s) for s in suffixes]):
	print("Don't know what to do with argument " + lib, file=sys.stderr)
	exit(1)
	libs.append(lib)

	# Check if calling convention decoration is used by inspecting the first
	# library in the list
	calling_convention_decoration = readobj_is_32bit_windows(args.readobj, libs[0])

	# Extract symbols from libraries in parallel. This is a huge time saver when
	# doing a debug build, as there are hundreds of thousands of symbols in each
	# library.
	# FIXME: On AIX, the default pool size can be too big for a logical
	# partition's allocated memory, and can lead to an out of memory
	# IO error. We are setting the pool size to 8 to avoid such
	# errors at the moment, and will look for a graceful solution later.
	pool = multiprocessing.Pool(8) if platform.system() == "AIX" \
	else multiprocessing.Pool()
	try:
	# Only one argument can be passed to the mapping function, and we can't
	# use a lambda or local function definition as that doesn't work on
	# windows, so create a list of tuples which duplicates the arguments
	# that are the same in all calls.
	vals = [
	(args.nm, should_keep_symbol, calling_convention_decoration, x)
	for x in libs
	]
	# Do an async map then wait for the result to make sure that
	# KeyboardInterrupt gets caught correctly (see
	# http://bugs.python.org/issue8296)
	result = pool.map_async(extract_symbols, vals)
	pool.close()
	libs_symbols = result.get(3600)
	except KeyboardInterrupt:
	# On Ctrl-C terminate everything and exit
	pool.terminate()
	pool.join()
	exit(1)

	# Merge everything into a single dict
	symbol_defs = dict()
	symbol_refs = set()
	for (this_lib_defs, this_lib_refs) in libs_symbols:
	for k, v in list(this_lib_defs.items()):
	symbol_defs[k] = v + symbol_defs.setdefault(k, 0)
	for sym in list(this_lib_refs):
	symbol_refs.add(sym)

	# Find which template instantiations are referenced at least once.
	template_instantiation_refs = set()
	for sym in list(symbol_refs):
	template = get_template_name(sym, args.mangling)
	if template:
	template_instantiation_refs.add(template)

	# Print symbols which both:
	# * Appear in exactly one input, as symbols defined in multiple
	# objects/libraries are assumed to have public definitions.
	# * Are not a template instantiation that isn't referenced anywhere. This
	# is because we need to export any explicitly instantiated templates,
	# and we expect those to be referenced in some object.
	if args.o:
	outfile = open(args.o, "w")
	else:
	outfile = sys.stdout
	for k, v in list(symbol_defs.items()):
	template = get_template_name(k, args.mangling)
	if v == 1 and (not template or template in template_instantiation_refs):
	print(k, file=outfile)