llvm/utils/UnicodeData/UnicodeCharSetsGenerator.cpp

//===--- UnicodeCharSetsGenerator.cpp - Unicode character sets generator --===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file is used to generate clang/lib/Lex/UnicodeCharSetsGenerated.cpp and
// llvm/lib/Support/UnicodeCharSetsGenerated.cpp from the XML representation of
// the Unicode Character Database available at
// https://www.unicode.org/Public/UCD/latest/ucdxml/
//
//===----------------------------------------------------------------------===//

#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/FileSystem.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/InitLLVM.h"
#include "llvm/Support/ToolOutputFile.h"
#include "llvm/Support/UnicodeCharRanges.h"
#include "llvm/Support/WithColor.h"

#include <libxml/xmlreader.h>
#include <vector>

using namespace llvm;

static cl::opt<std::string> UCDFile(cl::Positional, cl::Required,
                                    cl::desc("<UCD XML file>"));
static cl::opt<std::string> ClangOutput(cl::Positional, cl::Required,
                                        cl::desc("<clang output file>"));
static cl::opt<std::string> LLVMOutput(cl::Positional, cl::Required,
                                       cl::desc("<LLVM output file>"));

static constexpr unsigned CodePointCount = 0x110000;

static constexpr StringLiteral FileHeader = R"(
//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file was generated by llvm/utils/UnicodeData/UnicodeCharSetsGenerator
// from the Unicode {0} Character Database. Do not edit manually.
//
//===----------------------------------------------------------------------===//

#include "llvm/Support/UnicodeCharRanges.h"

using llvm::sys::UnicodeCharRange;
using CharRanges = llvm::sys::UnicodeCharSet::CharRanges;

namespace {1} {{
)";

[[noreturn]] static void error(const Twine &Message) {
  WithColor::error() << Message << '\n';
  exit(EXIT_FAILURE);
}

namespace {
struct Properties {
  unsigned XIDStart : 1;
  unsigned XIDContinue : 1;
  unsigned MathStart : 1;
  unsigned MathContinue : 1;
  unsigned Char : 1;
  unsigned Printable : 1;
  unsigned Control : 1;
  unsigned Format : 1;
  unsigned Combining : 1;
  unsigned DoubleWidth : 1;
};

struct CharacterDatabase {
  std::string Version;
  std::vector<Properties> Chars;
};

struct XmlDeleter {
  void operator()(xmlTextReader *Reader) { xmlFreeTextReader(Reader); }
  void operator()(xmlChar *Ptr) { xmlFree(Ptr); }
};
} // namespace

static StringRef fromXmlChar(const xmlChar *S) {
  return S ? StringRef(reinterpret_cast<const char *>(S)) : StringRef();
}

static bool isPrintableGC(StringRef GC) {
  return GC == "Zs" || (!GC.empty() && StringRef("LMNPS").contains(GC.front()));
}

// Loads the properties we care about from the flat XML representation of the
// UCD, which describes each code point, or range of code points, with all of
// its properties, e.g.
//   <char cp="0041" age="1.1" na="LATIN CAPITAL LETTER A" gc="Lu" ... />
static CharacterDatabase loadDatabase(StringRef Path) {
  std::unique_ptr<xmlTextReader, XmlDeleter> Reader(xmlReaderForFile(
      Path.str().c_str(), /*encoding=*/nullptr, XML_PARSE_NONET));
  if (!Reader)
    error(Path + ": cannot open file");

  CharacterDatabase Database;
  Database.Chars.resize(CodePointCount);
  int Status;
  while ((Status = xmlTextReaderRead(Reader.get())) == 1) {
    if (xmlTextReaderNodeType(Reader.get()) != XML_READER_TYPE_ELEMENT)
      continue;
    StringRef Element = fromXmlChar(xmlTextReaderConstLocalName(Reader.get()));
    if (Element == "description") {
      // <description>Unicode 18.0.0</description>
      std::unique_ptr<xmlChar, XmlDeleter> Description(
          xmlTextReaderReadString(Reader.get()));
      if (Description)
        Database.Version = fromXmlChar(Description.get())
                               .rsplit(' ')
                               .second.rsplit('.')
                               .first.str();
      continue;
    }
    if (!is_contained({"char", "reserved", "noncharacter", "surrogate"},
                      Element))
      continue;

    unsigned First = 0, Last = 0;
    bool HasRange = false;
    unsigned Found = 0;
    Properties P = {};
    while (xmlTextReaderMoveToNextAttribute(Reader.get()) == 1) {
      StringRef Name = fromXmlChar(xmlTextReaderConstLocalName(Reader.get()));
      StringRef Value = fromXmlChar(xmlTextReaderConstValue(Reader.get()));
      P.Char = Element == "char";
      if (Name == "cp" || Name == "first-cp") {
        if (Value.getAsInteger(16, First))
          break;
        if (Name == "cp")
          Last = First;
        HasRange = true;
      } else if (Name == "last-cp") {
        if (Value.getAsInteger(16, Last))
          HasRange = false;
      } else if (Name == "XIDS") {
        P.XIDStart = Value == "Y";
        ++Found;
      } else if (Name == "XIDC") {
        P.XIDContinue = Value == "Y";
        ++Found;
      } else if (Name == "ID_Compat_Math_Start") {
        P.MathStart = Value == "Y";
        ++Found;
      } else if (Name == "ID_Compat_Math_Continue") {
        P.MathContinue = Value == "Y";
        ++Found;
      } else if (Name == "gc") {
        P.Printable = isPrintableGC(Value);
        P.Control = Value == "Cc";
        P.Format = Value == "Cf";
        P.Combining = Value == "Mn" || Value == "Me";
        ++Found;
      } else if (Name == "ea") {
        P.DoubleWidth = Value == "F" || Value == "W";
        ++Found;
      }
    }

    if (!HasRange || First > Last || Last >= CodePointCount || Found != 6)
      error(Path + ":" + Twine(xmlTextReaderGetParserLineNumber(Reader.get())) +
            ": invalid <" + Element + "> element");
    llvm::fill(MutableArrayRef(Database.Chars).slice(First, Last - First + 1),
               P);
  }
  if (Status != 0)
    error(Path + ": failed to parse XML");
  if (Database.Version.empty())
    error(Path + ": missing Unicode version");
  return Database;
}

template <typename Predicate>
static void emitTable(raw_ostream &OS, StringRef Name, StringRef Description,
                      ArrayRef<Properties> Chars, Predicate Pred) {
  SmallVector<sys::UnicodeCharRange> Ranges;
  for (unsigned CodePoint = 0, E = Chars.size(); CodePoint != E; ++CodePoint) {
    if (!Pred(Chars[CodePoint]))
      continue;
    if (!Ranges.empty() && Ranges.back().Upper + 1 == CodePoint)
      Ranges.back().Upper = CodePoint;
    else
      Ranges.push_back({CodePoint, CodePoint});
  }

  OS << "\n// " << Description << "\nstatic constexpr UnicodeCharRange " << Name
     << "Data[] = {\n";
  for (sys::UnicodeCharRange Range : Ranges)
    OS << formatv("    {{{0:X4}, {1:X4}},\n", Range.Lower, Range.Upper);
  OS << "};\nextern const CharRanges " << Name << " = " << Name << "Data;\n";
}

static void writeFile(StringRef Path, StringRef Namespace, StringRef Version,
                      function_ref<void(raw_ostream &)> EmitTables) {
  std::error_code EC;
  ToolOutputFile Out(Path, EC, sys::fs::OF_Text);
  if (EC)
    error(Path + ": " + EC.message());

  Out.os() << formatv(FileHeader.drop_front().data(), Version, Namespace);
  EmitTables(Out.os());
  Out.os() << "\n} // namespace " << Namespace << '\n';
  Out.keep();
}

int main(int argc, char **argv) {
  InitLLVM X(argc, argv);
  cl::ParseCommandLineOptions(
      argc, argv,
      "Unicode character sets generator\n\n"
      "  Generates clang/lib/Lex/UnicodeCharSetsGenerated.cpp and\n"
      "  llvm/lib/Support/UnicodeCharSetsGenerated.cpp from the flat XML\n"
      "  representation of the Unicode Character Database, which can be\n"
      "  downloaded from https://www.unicode.org/Public/UCD/latest/ucdxml/\n"
      "  The generated files must then be formatted with clang-format.\n");

  CharacterDatabase UCD = loadDatabase(UCDFile);
  ArrayRef<Properties> Chars = UCD.Chars;

  writeFile(ClangOutput, "clang", UCD.Version, [&](raw_ostream &OS) {
    emitTable(OS, "GeneratedXIDStartRanges", "XID_Start", Chars,
              [](Properties P) { return P.XIDStart; });
    emitTable(OS, "GeneratedXIDContinueRanges",
              "XID_Continue, excluding XID_Start", Chars,
              [](Properties P) { return P.XIDContinue && !P.XIDStart; });
    emitTable(OS, "GeneratedMathematicalNotationProfileIDStartRanges",
              "ID_Compat_Math_Start", Chars,
              [](Properties P) { return P.MathStart; });
    emitTable(OS, "GeneratedMathematicalNotationProfileIDContinueRanges",
              "ID_Compat_Math_Continue, excluding ID_Compat_Math_Start", Chars,
              [](Properties P) { return P.MathContinue && !P.MathStart; });
  });
  writeFile(
      LLVMOutput, "llvm::sys::unicode", UCD.Version, [&](raw_ostream &OS) {
        emitTable(OS, "GeneratedCharacterRanges", "Characters", Chars,
                  [](Properties P) { return P.Char; });
        emitTable(OS, "GeneratedPrintableRanges",
                  "General_Category=L|M|N|P|S|Zs", Chars,
                  [](Properties P) { return P.Printable; });
        emitTable(OS, "GeneratedControlCharacterRanges", "General_Category=Cc",
                  Chars, [](Properties P) { return P.Control; });
        emitTable(OS, "GeneratedFormatCharacterRanges", "General_Category=Cf",
                  Chars, [](Properties P) { return P.Format; });
        emitTable(OS, "GeneratedCombiningCharacterRanges",
                  "General_Category=Mn|Me", Chars,
                  [](Properties P) { return P.Combining; });
        emitTable(OS, "GeneratedDoubleWidthCharacterRanges",
                  "East_Asian_Width=F|W", Chars,
                  [](Properties P) { return P.DoubleWidth; });
      });
}