tools/include-mapping/gen_std.py - llvm-project/clang - Git at Google

 #!/usr/bin/env python
 #===- gen_std.py -  ------------------------------------------*- python -*--===#
 #
 # Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
 # See https://llvm.org/LICENSE.txt for license information.
 # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
 #
 #===------------------------------------------------------------------------===#

 """gen_std.py is a tool to generate a lookup table (from qualified names to
 include headers) for C/C++ Standard Library symbols by parsing archived HTML
 files from cppreference.

 The generated files are located in clang/include/Tooling/Inclusions.

 Caveats and FIXMEs:
   - only symbols directly in "std" namespace are added, we should also add std's
     subnamespace symbols (e.g. chrono).
   - symbols with multiple variants or defined in multiple headers aren't added,
     e.g. std::move, std::swap

 Usage:
   1. Install BeautifulSoup dependency, see instruction:
        https://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-beautiful-soup
   2. Download cppreference offline HTML files (html_book_20220730.zip in Unofficial Release) at
        https://en.cppreference.com/w/Cppreference:Archives
   3. Unzip the zip file from step 2 (e.g., to a "cppreference" directory). You should
      get a "cppreference/reference" directory.
   4. Run the command:
        // Generate C++ symbols
        python3 gen_std.py -cppreference cppreference/reference -symbols=cpp > StdSymbolMap.inc
        // Generate C symbols
        python3 gen_std.py -cppreference cppreference/reference -symbols=c > CSymbolMap.inc
 """


 import cppreference_parser
 import argparse
 import datetime
 import os
 import sys
 import re


 CODE_PREFIX = """\
 //===-- gen_std.py generated file -------------------------------*- C++ -*-===//
 //
 // Used to build a lookup table (qualified names => include headers) for %s
 // Standard Library symbols.
 //
 // This file was generated automatically by
 // clang/tools/include-mapping/gen_std.py, DO NOT EDIT!
 //
 // Generated from cppreference offline HTML book (modified on %s).
 //===----------------------------------------------------------------------===//
 """

 def ParseArg():
   parser = argparse.ArgumentParser(description='Generate StdGen file')
   parser.add_argument('-cppreference', metavar='PATH',
                       default='',
                       help='path to the cppreference offline HTML directory',
                       required=True
                      )
   parser.add_argument('-symbols',
                       default='cpp',
                       help='Generate c or cpp (removed) symbols. One of {cpp, c, cpp_removed}.',
                       required=True)
   return parser.parse_args()

 def AdditionalHeadersForIOSymbols(symbol):
   # IO-related symbols declared in the <iosfwd> header, per C++
   # [iosfwd.syn 31.3.1]:
   iosfwd_symbols = [
       'basic_ios',
       'basic_streambuf',
       'basic_istream',
       'basic_ostream',
       'basic_iostream',

       'basic_stringbuf',
       'basic_istringstream',
       'basic_ostringstream',
       'basic_stringstream',

       'basic_spanbuf',
       'basic_ispanstream',
       'basic_ospanstream',
       'basic_spanstream',

       'basic_filebuf',
       'basic_ifstream',
       'basic_ofstream',
       'basic_fstream',

       'basic_syncbuf',
       'basic_osyncstream',

       'istreambuf_iterator',
       'ostreambuf_iterator',

       'ios',
       'wios',

       'streambuf',
       'istream',
       'ostream',
       'iostream',

       'stringbuf',
       'istringstream',
       'ostringstream',
       'stringstream',

       'spanbuf',
       'ispanstream',
       'ospanstream',
       'spanstream',

       'filebuf',
       'ifstream',
       'ofstream',
       'fstream',

       'syncbuf',
       'osyncstream',

       'wstreambuf',
       'wistream',
       'wostream',
       'wiostream',

       'wstringbuf',
       'wistringstream',
       'wostringstream',
       'wstringstream',

       'wspanbuf',
       'wispanstream',
       'wospanstream',
       'wspanstream',

       'wfilebuf',
       'wifstream',
       'wofstream',
       'wfstream',

       'wsyncbuf',
       'wosyncstream',

       'fpos',
       'streampos',
       'wstreampos',
       'u8streampos',
       'u16streampos',
       'u32streampos',
   ]
   assert(len(symbol.headers) == 1)
   sym_header = symbol.headers[0]
   headers = []
   # <iostream> is preferred than <iosfwd>

   # <iostream> is an alternative of <streambuf>, <istream>, <ostream>, <ios>.
   # per C++ [iostream.syn 31.4.1]
   if sym_header in ["<ios>", "<istream>", "<ostream>", "<streambuf>"]:
     headers.append("<iostream>")

   if symbol.name in iosfwd_symbols:
     headers.append("<iosfwd>")

   return headers


 def GetCCompatibilitySymbols(symbol):
    # C++ form of the C standard headers.
   c_compat_headers = {
     "<cassert>",
     "<cctype>",
     "<cerrno>",
     "<cfenv>",
     "<cfloat>",
     "<cinttypes>",
     "<climits>",
     "<clocale>",
     "<cmath>",
     "<csetjmp>",
     "<csignal>",
     "<cstdarg>",
     "<cstddef>",
     "<cstdint>",
     "<cstdio>",
     "<cstdlib>",
     "<cstring>",
     "<ctime>",
     "<cuchar>",
     "<cwchar>",
     "<cwctype>",
   }
   # C++ [support.c.headers.other] 17.14.7
   #    ..., behaves as if each name placed in the standard library namespace by
   #    the corresponding <cname> header is placed within the global namespace
   #    scope, except for the functions described in [sf.cmath], the
   #    std::lerp function overloads ([c.math.lerp]), the declaration of
   #    std::byte ([cstddef.syn]), and the functions and function templates
   #    described in [support.types.byteops].
   exception_symbols = {
     "(assoc_)?laguerre[f|l]?",
     "(assoc_|sph_)?legendre[f|l]?",
     "beta[f|l]?",
     "(comp_)?ellint_[1-3][f|l]?",
     "(cyl_|sph_)?bessel_[i-k][f|l]?",
     "(cyl_|sph_)?neumann[f|l]?",
     "expint[f|l]?",
     "hermite[f|l]?",
     "riemann_zeta[f|l]?",
     "lerp",
     "byte",
   }
   assert(len(symbol.headers) == 1)
   header = symbol.headers[0]
   if header not in c_compat_headers:
     return []
   if any(re.fullmatch(x, symbol.name) for x in exception_symbols):
     return []

   # Introduce two more entries, both in the global namespace, one using the
   # C++-compat header and another using the C header.
   results = []
   if symbol.namespace != None:
     # avoid printing duplicated entries, for C macros!
     results.append(cppreference_parser.Symbol(symbol.name, None, [header]))
   c_header = "<" + header[2:-1] +  ".h>" # <cstdio> => <stdio.h>
   results.append(cppreference_parser.Symbol(symbol.name, None, [c_header]))
   return results


 def main():
   args = ParseArg()
   if args.symbols == 'cpp':
     page_root = os.path.join(args.cppreference, "en", "cpp")
     symbol_index_root = os.path.join(page_root, "symbol_index")
     parse_pages =  [
       (page_root, "symbol_index.html", "std::"),
       # std sub-namespace symbols have separated pages.
       # We don't index std literal operators (e.g.
       # std::literals::chrono_literals::operator""d), these symbols can't be
       # accessed by std::<symbol_name>.
       #
       # std::placeholders symbols are handled manually in StdSpecialSymbolMap.inc
       (symbol_index_root, "chrono.html", "std::chrono::"),
       (symbol_index_root, "execution.html", "std::execution::"),
       (symbol_index_root, "numbers.html", "std::numbers::"),
       (symbol_index_root, "filesystem.html", "std::filesystem::"),
       (symbol_index_root, "pmr.html", "std::pmr::"),
       (symbol_index_root, "ranges.html", "std::ranges::"),
       (symbol_index_root, "regex_constants.html", "std::regex_constants::"),
       (symbol_index_root, "this_thread.html", "std::this_thread::"),
       # Zombie symbols that were available from the Standard Library, but are
       # removed in the following standards.
       (symbol_index_root, "zombie_names.html", "std::"),
       (symbol_index_root, "macro.html", None),
     ]
   elif args.symbols == 'c':
     page_root = os.path.join(args.cppreference, "en", "c")
     symbol_index_root = page_root
     parse_pages = [(page_root, "index.html", None)]

   if not os.path.exists(symbol_index_root):
     exit("Path %s doesn't exist!" % symbol_index_root)

   symbols = cppreference_parser.GetSymbols(parse_pages)

   # We don't have version information from the unzipped offline HTML files.
   # so we use the modified time of the symbol_index.html as the version.
   index_page_path = os.path.join(page_root, "index.html")
   cppreference_modified_date = datetime.datetime.fromtimestamp(
     os.stat(index_page_path).st_mtime).strftime('%Y-%m-%d')
   print(CODE_PREFIX % (args.symbols.upper(), cppreference_modified_date))
   for symbol in symbols:
     if len(symbol.headers) == 1:
       augmented_symbols = [symbol]
       augmented_symbols.extend(GetCCompatibilitySymbols(symbol))
       for s in augmented_symbols:
         s.headers.extend(AdditionalHeadersForIOSymbols(s))
         for header in s.headers:
           # SYMBOL(unqualified_name, namespace, header)
           print("SYMBOL(%s, %s, %s)" % (s.name, s.namespace,
                                         header))
     elif len(symbol.headers) == 0:
       sys.stderr.write("No header found for symbol %s\n" % symbol.name)
     else:
       # FIXME: support symbols with multiple headers (e.g. std::move).
       sys.stderr.write("Ambiguous header for symbol %s: %s\n" % (
           symbol.name, ', '.join(symbol.headers)))


 if __name__ == '__main__':
   main()
	#!/usr/bin/env python
	#===- gen_std.py - ------------------------------------------- python ---===#
	#
	# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
	# See https://llvm.org/LICENSE.txt for license information.
	# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
	#
	#===------------------------------------------------------------------------===#

	"""gen_std.py is a tool to generate a lookup table (from qualified names to
	include headers) for C/C++ Standard Library symbols by parsing archived HTML
	files from cppreference.

	The generated files are located in clang/include/Tooling/Inclusions.

	Caveats and FIXMEs:
	- only symbols directly in "std" namespace are added, we should also add std's
	subnamespace symbols (e.g. chrono).
	- symbols with multiple variants or defined in multiple headers aren't added,
	e.g. std::move, std::swap

	Usage:
	1. Install BeautifulSoup dependency, see instruction:
	https://www.crummy.com/software/BeautifulSoup/bs4/doc/#installing-beautiful-soup
	2. Download cppreference offline HTML files (html_book_20220730.zip in Unofficial Release) at
	https://en.cppreference.com/w/Cppreference:Archives
	3. Unzip the zip file from step 2 (e.g., to a "cppreference" directory). You should
	get a "cppreference/reference" directory.
	4. Run the command:
	// Generate C++ symbols
	python3 gen_std.py -cppreference cppreference/reference -symbols=cpp > StdSymbolMap.inc
	// Generate C symbols
	python3 gen_std.py -cppreference cppreference/reference -symbols=c > CSymbolMap.inc
	"""


	import cppreference_parser
	import argparse
	import datetime
	import os
	import sys
	import re


	CODE_PREFIX = """\
	//===-- gen_std.py generated file -------------------------------- C++ --===//
	//
	// Used to build a lookup table (qualified names => include headers) for %s
	// Standard Library symbols.
	//
	// This file was generated automatically by
	// clang/tools/include-mapping/gen_std.py, DO NOT EDIT!
	//
	// Generated from cppreference offline HTML book (modified on %s).
	//===----------------------------------------------------------------------===//
	"""

	def ParseArg():
	parser = argparse.ArgumentParser(description='Generate StdGen file')
	parser.add_argument('-cppreference', metavar='PATH',
	default='',
	help='path to the cppreference offline HTML directory',
	required=True
	)
	parser.add_argument('-symbols',
	default='cpp',
	help='Generate c or cpp (removed) symbols. One of {cpp, c, cpp_removed}.',
	required=True)
	return parser.parse_args()

	def AdditionalHeadersForIOSymbols(symbol):
	# IO-related symbols declared in the <iosfwd> header, per C++
	# [iosfwd.syn 31.3.1]:
	iosfwd_symbols = [
	'basic_ios',
	'basic_streambuf',
	'basic_istream',
	'basic_ostream',
	'basic_iostream',

	'basic_stringbuf',
	'basic_istringstream',
	'basic_ostringstream',
	'basic_stringstream',

	'basic_spanbuf',
	'basic_ispanstream',
	'basic_ospanstream',
	'basic_spanstream',

	'basic_filebuf',
	'basic_ifstream',
	'basic_ofstream',
	'basic_fstream',

	'basic_syncbuf',
	'basic_osyncstream',

	'istreambuf_iterator',
	'ostreambuf_iterator',

	'ios',
	'wios',

	'streambuf',
	'istream',
	'ostream',
	'iostream',

	'stringbuf',
	'istringstream',
	'ostringstream',
	'stringstream',

	'spanbuf',
	'ispanstream',
	'ospanstream',
	'spanstream',

	'filebuf',
	'ifstream',
	'ofstream',
	'fstream',

	'syncbuf',
	'osyncstream',

	'wstreambuf',
	'wistream',
	'wostream',
	'wiostream',

	'wstringbuf',
	'wistringstream',
	'wostringstream',
	'wstringstream',

	'wspanbuf',
	'wispanstream',
	'wospanstream',
	'wspanstream',

	'wfilebuf',
	'wifstream',
	'wofstream',
	'wfstream',

	'wsyncbuf',
	'wosyncstream',

	'fpos',
	'streampos',
	'wstreampos',
	'u8streampos',
	'u16streampos',
	'u32streampos',
	]
	assert(len(symbol.headers) == 1)
	sym_header = symbol.headers[0]
	headers = []
	# <iostream> is preferred than <iosfwd>

	# <iostream> is an alternative of <streambuf>, <istream>, <ostream>, <ios>.
	# per C++ [iostream.syn 31.4.1]
	if sym_header in ["<ios>", "<istream>", "<ostream>", "<streambuf>"]:
	headers.append("<iostream>")

	if symbol.name in iosfwd_symbols:
	headers.append("<iosfwd>")

	return headers


	def GetCCompatibilitySymbols(symbol):
	# C++ form of the C standard headers.
	c_compat_headers = {
	"<cassert>",
	"<cctype>",
	"<cerrno>",
	"<cfenv>",
	"<cfloat>",
	"<cinttypes>",
	"<climits>",
	"<clocale>",
	"<cmath>",
	"<csetjmp>",
	"<csignal>",
	"<cstdarg>",
	"<cstddef>",
	"<cstdint>",
	"<cstdio>",
	"<cstdlib>",
	"<cstring>",
	"<ctime>",
	"<cuchar>",
	"<cwchar>",
	"<cwctype>",
	}
	# C++ [support.c.headers.other] 17.14.7
	# ..., behaves as if each name placed in the standard library namespace by
	# the corresponding <cname> header is placed within the global namespace
	# scope, except for the functions described in [sf.cmath], the
	# std::lerp function overloads ([c.math.lerp]), the declaration of
	# std::byte ([cstddef.syn]), and the functions and function templates
	# described in [support.types.byteops].
	exception_symbols = {
	"(assoc_)?laguerre[f\|l]?",
	"(assoc_\|sph_)?legendre[f\|l]?",
	"beta[f\|l]?",
	"(comp_)?ellint_[1-3][f\|l]?",
	"(cyl_\|sph_)?bessel_[i-k][f\|l]?",
	"(cyl_\|sph_)?neumann[f\|l]?",
	"expint[f\|l]?",
	"hermite[f\|l]?",
	"riemann_zeta[f\|l]?",
	"lerp",
	"byte",
	}
	assert(len(symbol.headers) == 1)
	header = symbol.headers[0]
	if header not in c_compat_headers:
	return []
	if any(re.fullmatch(x, symbol.name) for x in exception_symbols):
	return []

	# Introduce two more entries, both in the global namespace, one using the
	# C++-compat header and another using the C header.
	results = []
	if symbol.namespace != None:
	# avoid printing duplicated entries, for C macros!
	results.append(cppreference_parser.Symbol(symbol.name, None, [header]))
	c_header = "<" + header[2:-1] + ".h>" # <cstdio> => <stdio.h>
	results.append(cppreference_parser.Symbol(symbol.name, None, [c_header]))
	return results


	def main():
	args = ParseArg()
	if args.symbols == 'cpp':
	page_root = os.path.join(args.cppreference, "en", "cpp")
	symbol_index_root = os.path.join(page_root, "symbol_index")
	parse_pages = [
	(page_root, "symbol_index.html", "std::"),
	# std sub-namespace symbols have separated pages.
	# We don't index std literal operators (e.g.
	# std::literals::chrono_literals::operator""d), these symbols can't be
	# accessed by std::<symbol_name>.
	#
	# std::placeholders symbols are handled manually in StdSpecialSymbolMap.inc
	(symbol_index_root, "chrono.html", "std::chrono::"),
	(symbol_index_root, "execution.html", "std::execution::"),
	(symbol_index_root, "numbers.html", "std::numbers::"),
	(symbol_index_root, "filesystem.html", "std::filesystem::"),
	(symbol_index_root, "pmr.html", "std::pmr::"),
	(symbol_index_root, "ranges.html", "std::ranges::"),
	(symbol_index_root, "regex_constants.html", "std::regex_constants::"),
	(symbol_index_root, "this_thread.html", "std::this_thread::"),
	# Zombie symbols that were available from the Standard Library, but are
	# removed in the following standards.
	(symbol_index_root, "zombie_names.html", "std::"),
	(symbol_index_root, "macro.html", None),
	]
	elif args.symbols == 'c':
	page_root = os.path.join(args.cppreference, "en", "c")
	symbol_index_root = page_root
	parse_pages = [(page_root, "index.html", None)]

	if not os.path.exists(symbol_index_root):
	exit("Path %s doesn't exist!" % symbol_index_root)

	symbols = cppreference_parser.GetSymbols(parse_pages)

	# We don't have version information from the unzipped offline HTML files.
	# so we use the modified time of the symbol_index.html as the version.
	index_page_path = os.path.join(page_root, "index.html")
	cppreference_modified_date = datetime.datetime.fromtimestamp(
	os.stat(index_page_path).st_mtime).strftime('%Y-%m-%d')
	print(CODE_PREFIX % (args.symbols.upper(), cppreference_modified_date))
	for symbol in symbols:
	if len(symbol.headers) == 1:
	augmented_symbols = [symbol]
	augmented_symbols.extend(GetCCompatibilitySymbols(symbol))
	for s in augmented_symbols:
	s.headers.extend(AdditionalHeadersForIOSymbols(s))
	for header in s.headers:
	# SYMBOL(unqualified_name, namespace, header)
	print("SYMBOL(%s, %s, %s)" % (s.name, s.namespace,
	header))
	elif len(symbol.headers) == 0:
	sys.stderr.write("No header found for symbol %s\n" % symbol.name)
	else:
	# FIXME: support symbols with multiple headers (e.g. std::move).
	sys.stderr.write("Ambiguous header for symbol %s: %s\n" % (
	symbol.name, ', '.join(symbol.headers)))


	if __name__ == '__main__':
	main()